From eef117bb790ec3ac347585b6ec90a6bdfcbd5626 Mon Sep 17 00:00:00 2001 From: fzowl Date: Fri, 27 Dec 2024 22:04:02 +0100 Subject: [PATCH 001/411] Refresh VoyageAI models and prices and context --- model_prices_and_context_window.json | 94 ++++++++++++++++++---------- 1 file changed, 62 insertions(+), 32 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5f98c0e68b0..a12d114f952 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7566,30 +7566,6 @@ "litellm_provider": "cloudflare", "mode": "chat" }, - "voyage/voyage-01": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, - "voyage/voyage-lite-01": { - "max_tokens": 4096, - "max_input_tokens": 4096, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, - "voyage/voyage-large-2": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "input_cost_per_token": 0.00000012, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, "voyage/voyage-law-2": { "max_tokens": 16000, "max_input_tokens": 16000, @@ -7614,14 +7590,6 @@ "litellm_provider": "voyage", "mode": "embedding" }, - "voyage/voyage-lite-02-instruct": { - "max_tokens": 4000, - "max_input_tokens": 4000, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, "voyage/voyage-finance-2": { "max_tokens": 4000, "max_input_tokens": 4000, @@ -7630,6 +7598,68 @@ "litellm_provider": "voyage", "mode": "embedding" }, + "voyage/voyage-3-large": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000018, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-3": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000006, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-3-lite": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000002, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-code-3": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000018, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-multimodal-3": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000012, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/rerank-2": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "max_query_tokens": 16000, + "input_cost_per_token": 0.00000005, + "input_cost_per_query": 0.00000005, + "output_cost_per_token": 0.0, + "litellm_provider": "voyage", + "mode": "rerank" + }, + "voyage/rerank-2-lite": { + "max_tokens": 8000, + "max_input_tokens": 8000, + "max_output_tokens": 8000, + "max_query_tokens": 8000, + "input_cost_per_token": 0.00000002, + "input_cost_per_query": 0.00000002, + "output_cost_per_token": 0.0, + "litellm_provider": "voyage", + "mode": "rerank" + }, "databricks/databricks-meta-llama-3-1-405b-instruct": { "max_tokens": 128000, "max_input_tokens": 128000, From 6876a14838605865276ac17ea9991e1874f8d64a Mon Sep 17 00:00:00 2001 From: fzowl Date: Mon, 30 Dec 2024 11:50:37 +0100 Subject: [PATCH 002/411] Refresh VoyageAI models and prices and context --- model_prices_and_context_window.json | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2ff5e40247d..0701b20af49 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7613,6 +7613,38 @@ "litellm_provider": "cloudflare", "mode": "chat" }, + "voyage/voyage-01": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-lite-01": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-large-2": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "input_cost_per_token": 0.00000012, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, + "voyage/voyage-lite-02-instruct": { + "max_tokens": 4000, + "max_input_tokens": 4000, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, "voyage/voyage-law-2": { "max_tokens": 16000, "max_input_tokens": 16000, From bb0f74c48f9a85874f2f104a6de0f043d4658ea6 Mon Sep 17 00:00:00 2001 From: fzowl Date: Mon, 30 Dec 2024 11:54:37 +0100 Subject: [PATCH 003/411] Refresh VoyageAI models and prices and context --- model_prices_and_context_window.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0701b20af49..1a9ed533e38 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7637,6 +7637,14 @@ "litellm_provider": "voyage", "mode": "embedding" }, + "voyage/voyage-finance-2": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "input_cost_per_token": 0.00000012, + "output_cost_per_token": 0.000000, + "litellm_provider": "voyage", + "mode": "embedding" + }, "voyage/voyage-lite-02-instruct": { "max_tokens": 4000, "max_input_tokens": 4000, @@ -7669,14 +7677,6 @@ "litellm_provider": "voyage", "mode": "embedding" }, - "voyage/voyage-finance-2": { - "max_tokens": 4000, - "max_input_tokens": 4000, - "input_cost_per_token": 0.00000012, - "output_cost_per_token": 0.000000, - "litellm_provider": "voyage", - "mode": "embedding" - }, "voyage/voyage-3-large": { "max_tokens": 32000, "max_input_tokens": 32000, From 9690cb66c6e51f81db586152d822965e1db94a60 Mon Sep 17 00:00:00 2001 From: fzowl Date: Mon, 3 Feb 2025 15:21:20 +0100 Subject: [PATCH 004/411] Updating the available VoyageAI models in the docs --- docs/my-website/docs/providers/voyage.md | 27 +++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/my-website/docs/providers/voyage.md b/docs/my-website/docs/providers/voyage.md index a56a1408ea9..6ab6b1846f5 100644 --- a/docs/my-website/docs/providers/voyage.md +++ b/docs/my-website/docs/providers/voyage.md @@ -14,7 +14,7 @@ import os os.environ['VOYAGE_API_KEY'] = "" response = embedding( - model="voyage/voyage-01", + model="voyage/voyage-3-large", input=["good morning from litellm"], ) print(response) @@ -23,13 +23,20 @@ print(response) ## Supported Models All models listed here https://docs.voyageai.com/embeddings/#models-and-specifics are supported -| Model Name | Function Call | -|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| voyage-2 | `embedding(model="voyage/voyage-2", input)` | -| voyage-large-2 | `embedding(model="voyage/voyage-large-2", input)` | -| voyage-law-2 | `embedding(model="voyage/voyage-law-2", input)` | -| voyage-code-2 | `embedding(model="voyage/voyage-code-2", input)` | +| Model Name | Function Call | +|-------------------------|------------------------------------------------------------| +| voyage-3-large | `embedding(model="voyage/voyage-3-large", input)` | +| voyage-3 | `embedding(model="voyage/voyage-3", input)` | +| voyage-3-lite | `embedding(model="voyage/voyage-3-lite", input)` | +| voyage-code-3 | `embedding(model="voyage/voyage-code-3", input)` | +| voyage-finance-2 | `embedding(model="voyage/voyage-finance-2", input)` | +| voyage-law-2 | `embedding(model="voyage/voyage-law-2", input)` | +| voyage-code-2 | `embedding(model="voyage/voyage-code-2", input)` | +| voyage-multilingual-2 | `embedding(model="voyage/voyage-multilingual-2 ", input)` | +| voyage-large-2-instruct | `embedding(model="voyage/voyage-large-2-instruct", input)` | +| voyage-large-2 | `embedding(model="voyage/voyage-large-2", input)` | +| voyage-2 | `embedding(model="voyage/voyage-2", input)` | | voyage-lite-02-instruct | `embedding(model="voyage/voyage-lite-02-instruct", input)` | -| voyage-01 | `embedding(model="voyage/voyage-01", input)` | -| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` | -| voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` | \ No newline at end of file +| voyage-01 | `embedding(model="voyage/voyage-01", input)` | +| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` | +| voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` | From 818db5e59032f37bd8f8440fd064607ae3f4c924 Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 21 May 2025 13:09:42 +0200 Subject: [PATCH 005/411] Updating the available VoyageAI models in the docs --- docs/my-website/docs/providers/voyage.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/providers/voyage.md b/docs/my-website/docs/providers/voyage.md index 6ab6b1846f5..4b729bc9f58 100644 --- a/docs/my-website/docs/providers/voyage.md +++ b/docs/my-website/docs/providers/voyage.md @@ -25,6 +25,8 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific | Model Name | Function Call | |-------------------------|------------------------------------------------------------| +| voyage-3.5 | `embedding(model="voyage/voyage-3.5", input)` | +| voyage-3.5-lite | `embedding(model="voyage/voyage-3.5-lite", input)` | | voyage-3-large | `embedding(model="voyage/voyage-3-large", input)` | | voyage-3 | `embedding(model="voyage/voyage-3", input)` | | voyage-3-lite | `embedding(model="voyage/voyage-3-lite", input)` | @@ -35,8 +37,8 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific | voyage-multilingual-2 | `embedding(model="voyage/voyage-multilingual-2 ", input)` | | voyage-large-2-instruct | `embedding(model="voyage/voyage-large-2-instruct", input)` | | voyage-large-2 | `embedding(model="voyage/voyage-large-2", input)` | -| voyage-2 | `embedding(model="voyage/voyage-2", input)` | +| voyage-2 | `embedding(model="voyage/voyage-2", input)` | | voyage-lite-02-instruct | `embedding(model="voyage/voyage-lite-02-instruct", input)` | -| voyage-01 | `embedding(model="voyage/voyage-01", input)` | -| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` | +| voyage-01 | `embedding(model="voyage/voyage-01", input)` | +| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` | | voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` | From bfc7ae05bf78982b7cce01e5af37ad4b6e8574b5 Mon Sep 17 00:00:00 2001 From: unknown <> Date: Wed, 1 Jul 2026 14:20:26 +0000 Subject: [PATCH 006/411] feat: add claude-sonnet-5 pricing to model cost map Add Claude Sonnet 5 with introductory pricing (/0 per MTok, through Aug 31 2026) across Anthropic, Bedrock (global + regional), Vertex AI, Azure AI, and Snowflake providers. Includes cache pricing (1.25x write, 2x 1hr write, 0.1x read) and 128K output context. Fixes #31868 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + ...odel_prices_and_context_window_backup.json | 1836 +++++++++++------ model_prices_and_context_window.json | 1398 +++++++------ .../test_claude_sonnet_5_config.py | 163 ++ 4 files changed, 2137 insertions(+), 1261 deletions(-) create mode 100644 tests/test_litellm/test_claude_sonnet_5_config.py diff --git a/litellm/constants.py b/litellm/constants.py index aeb74a65839..1d17e7b6da6 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1128,6 +1128,7 @@ BEDROCK_CONVERSE_MODELS = [ "anthropic.claude-opus-4-6-v1:0", "anthropic.claude-opus-4-6-v1", "anthropic.claude-sonnet-4-6", + "anthropic.claude-sonnet-5", "anthropic.claude-opus-4-1-20250805-v1:0", "anthropic.claude-opus-4-20250514-v1:0", "anthropic.claude-sonnet-4-20250514-v1:0", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 21132db93cb..15b16118a7b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1010,7 +1010,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1042,7 +1041,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1074,7 +1072,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1106,7 +1103,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1138,7 +1134,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1170,7 +1165,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1219,7 +1213,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1253,7 +1246,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1287,7 +1279,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1321,7 +1312,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1487,7 +1477,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1521,7 +1510,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1555,7 +1543,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1589,7 +1576,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1623,7 +1609,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1688,7 +1673,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1719,7 +1703,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1750,7 +1733,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1781,7 +1763,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1812,7 +1793,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1843,7 +1823,186 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "anthropic.claude-sonnet-5": { "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "global.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "us.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "eu.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "au.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "jp.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -2364,7 +2523,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2394,7 +2552,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2455,7 +2612,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2523,6 +2679,34 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, + "azure_ai/claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, @@ -9132,17 +9316,16 @@ }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -9754,17 +9937,16 @@ }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -10274,6 +10456,35 @@ "supports_vision": true, "supports_output_config": true }, + "claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -16134,6 +16345,46 @@ "tpm": 8000000, "supports_image_size": false }, + "gemini-3-pro-image": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_image_token": 0.00012, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3-pro-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -16174,6 +16425,44 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.1-flash-image": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, @@ -17588,6 +17877,48 @@ }, "supports_image_size": false }, + "gemini/gemini-3-pro-image": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_image_token": 0.00012, + "output_cost_per_token": 1.2e-05, + "rpm": 1000, + "tpm": 4000000, + "output_cost_per_token_batches": 6e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3-pro-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -17630,6 +17961,47 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.1-flash-image": { + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.045, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "rpm": 1000, + "tpm": 4000000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-3.1-flash-image-preview": { "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -18944,7 +19316,6 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_adaptive_thinking": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true @@ -21696,8 +22067,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21745,8 +22116,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21790,8 +22161,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21835,8 +22206,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21884,8 +22255,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21932,8 +22303,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21973,8 +22344,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22017,8 +22388,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22054,7 +22425,7 @@ "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -22062,8 +22433,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22100,7 +22471,7 @@ "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -22108,8 +22479,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22144,15 +22515,15 @@ "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22187,15 +22558,15 @@ "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22227,9 +22598,9 @@ "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 272000, - "max_tokens": 272000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -22263,9 +22634,9 @@ "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 272000, - "max_tokens": 272000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -24712,14 +25083,13 @@ }, "minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.2e-06, "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -28175,7 +28545,6 @@ "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, "source": "https://openrouter.ai/anthropic/claude-sonnet-4.6", - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28215,7 +28584,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28277,7 +28645,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -29522,6 +29889,22 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/z-ai/glm-5.1": { + "input_cost_per_token": 1.05e-06, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 5.25e-07, + "cache_creation_input_token_cost": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 202752, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.1", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/minimax/minimax-m2.1": { "input_cost_per_token": 2.7e-07, "output_cost_per_token": 1.2e-06, @@ -30200,7 +30583,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -30210,7 +30592,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -31126,7 +31507,7 @@ "supports_tool_choice": true }, "sambanova/Meta-Llama-3.2-1B-Instruct": { - "deprecation_date": "2025-06-25", + "deprecation_date": "2025-06-25", "input_cost_per_token": 4e-08, "litellm_provider": "sambanova", "max_input_tokens": 16384, @@ -31257,15 +31638,15 @@ "supports_vision": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "snowflake/claude-3-5-sonnet": { + "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 200000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, "supports_computer_use": true, "supports_function_calling": true, "supports_vision": true, @@ -31273,14 +31654,14 @@ "supports_system_messages": true, "supports_response_schema": true }, - "snowflake/deepseek-r1": { + "snowflake/deepseek-r1": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 5.4e-06, "supports_reasoning": true, "supports_system_messages": true }, @@ -31339,8 +31720,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.0000012, - "output_cost_per_token": 0.0000012, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 1.2e-06, "supports_function_calling": true, "supports_system_messages": true }, @@ -31350,8 +31731,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true }, @@ -31361,8 +31742,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000024, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 2.4e-07, "supports_system_messages": true }, "snowflake/llama3.2-1b": { @@ -31379,17 +31760,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/llama3.3-70b": { + "snowflake/llama3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "snowflake/mistral-7b": { "litellm_provider": "snowflake", "max_input_tokens": 32000, @@ -31404,14 +31785,14 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/mistral-large2": { + "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_response_schema": true @@ -31451,17 +31832,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/snowflake-llama-3.3-70b": { + "snowflake/snowflake-llama-3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "stability/sd3": { "litellm_provider": "stability", "mode": "image_generation", @@ -32713,7 +33094,7 @@ "input_cost_per_token": 3.6e-06, "input_cost_per_token_above_200k_tokens": 7.2e-06, "output_cost_per_token_above_200k_tokens": 2.7e-05, - "cache_creation_input_token_cost_above_200k_tokens": 9.0e-06, + "cache_creation_input_token_cost_above_200k_tokens": 9e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "litellm_provider": "bedrock_converse", @@ -33455,7 +33836,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -34690,7 +35070,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34720,7 +35099,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34750,7 +35128,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34781,7 +35158,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34872,7 +35248,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34903,7 +35278,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34956,7 +35330,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -35265,6 +35638,21 @@ "tpm": 8000000, "supports_image_size": false }, + "vertex_ai/gemini-3-pro-image": { + "input_cost_per_image": 0.0011, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.134, + "output_cost_per_image_token": 0.00012, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" + }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -35280,6 +35668,19 @@ "output_cost_per_token_batches": 6e-06, "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, + "vertex_ai/gemini-3.1-flash-image": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" + }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, @@ -37485,12 +37886,12 @@ }, "zai.glm-5": { "input_cost_per_token": 1e-06, + "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3.2e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -37511,20 +37912,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, @@ -37540,6 +37927,21 @@ "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing" }, + "zai/glm-5.1": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, "zai/glm-5-code": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 3e-07, @@ -37570,6 +37972,21 @@ "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing" }, + "zai/glm-4.7-flash": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 0, + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, "zai/glm-4.6": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 1.1e-07, @@ -42393,7 +42810,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -42411,6 +42827,64 @@ }, "supports_output_config": true }, + "vertex_ai/claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, + "vertex_ai/claude-sonnet-5@default": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, "duckduckgo/search": { "litellm_provider": "duckduckgo", "mode": "search", @@ -42427,7 +42901,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42442,7 +42919,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42457,7 +42937,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42471,7 +42953,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42487,9 +42971,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42507,9 +42998,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42526,7 +43024,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42542,7 +43043,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42558,7 +43062,10 @@ "max_tokens": 128000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42717,20 +43224,6 @@ } ] }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, @@ -42759,45 +43252,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-east-1/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-west-2/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, "cache_creation_input_token_cost_above_1hr": 2.4e-06, @@ -42844,362 +43298,382 @@ "supports_native_structured_output": true, "supports_pdf_input": true }, - "snowflake/claude-sonnet-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-sonnet-4-6": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-opus": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/claude-haiku-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 0.0000001, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-3-7-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-4.1": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.000000125, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-mini": { - "max_tokens": 16384, - "max_input_tokens": 1000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000012, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-nano": { - "max_tokens": 16384, - "max_input_tokens": 5000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.0000006, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/llama4-maverick": { - "max_tokens": 16384, - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000097, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true - }, - "snowflake/snowflake-arctic-embed-l-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "snowflake/snowflake-arctic-embed-m-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "soniox/stt-async-v4": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "soniox/stt-async-v5": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 6e-07, - "output_cost_per_token": 3.6e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 4.5e-07, - "output_cost_per_token": 1.8e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3.6-27B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3.2e-07, - "output_cost_per_token": 3.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 202752, - "max_output_tokens": 202752, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/moonshotai/Kimi-K2.6": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 9.6e-07, - "output_cost_per_token": 4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/MiniMaxAI/MiniMax-M2.5": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/google/gemma-4-31B-it": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 5.6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-120b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-20b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 7e-08, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - } - , + "snowflake/claude-sonnet-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-sonnet-4-6": { + "supports_adaptive_thinking": true, + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-sonnet-5": { + "supports_adaptive_thinking": true, + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-opus": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/claude-haiku-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-3-7-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-4.1": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-mini": { + "max_tokens": 16384, + "max_input_tokens": 1000000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-nano": { + "max_tokens": 16384, + "max_input_tokens": 5000000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/llama4-maverick": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9.7e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true + }, + "snowflake/snowflake-arctic-embed-l-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "snowflake/snowflake-arctic-embed-m-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "soniox/stt-async-v4": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "soniox/stt-async-v5": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 6e-07, + "output_cost_per_token": 3.6e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 1.8e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3.6-27B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 3.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 202752, + "max_output_tokens": 202752, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/moonshotai/Kimi-K2.6": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 9.6e-07, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/MiniMaxAI/MiniMax-M2.5": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/google/gemma-4-31B-it": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 5.6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-120b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-20b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 2.8e-09, @@ -43275,6 +43749,109 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek/deepseek-v4-pro": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 3.625e-09, + "input_cost_per_token": 4.35e-07, + "input_cost_per_token_cache_hit": 3.625e-09, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "pinstripes/ps/glm-4.5-air": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 4.5e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": true, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/qwen3.6-35b-a3b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 4.5e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": true, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/qwen3-30b-a3b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 2e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": true, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/qwen3-coder-30b-a3b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": false, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/deepseek-v4-flash": { + "max_tokens": 163840, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": true, + "source": "https://pinstripes.io/pricing" + }, + "pinstripes/ps/minimax-m2.7": { + "max_tokens": 1000192, + "max_input_tokens": 1000192, + "max_output_tokens": 1000192, + "input_cost_per_token": 2.55e-07, + "output_cost_per_token": 5.5e-07, + "litellm_provider": "pinstripes", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_reasoning": false, + "source": "https://pinstripes.io/pricing" + }, "darkbloom/gemma-4-26b": { "input_cost_per_token": 3e-08, "litellm_provider": "darkbloom", @@ -43309,142 +43886,39 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "deepseek/deepseek-v4-pro": { - "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 3.625e-09, - "input_cost_per_token": 4.35e-07, - "input_cost_per_token_cache_hit": 3.625e-09, - "litellm_provider": "deepseek", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8.7e-07, - "source": "https://api-docs.deepseek.com/quick_start/pricing", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "pinstripes/ps/glm-4.5-air": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.00000045, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/qwen3.6-35b-a3b": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.00000014, - "output_cost_per_token": 0.00000045, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/qwen3-30b-a3b": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.00000009, - "output_cost_per_token": 0.0000002, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/qwen3-coder-30b-a3b": { - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000006, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/deepseek-v4-flash": { - "max_tokens": 163840, - "max_input_tokens": 163840, - "max_output_tokens": 163840, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000002, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" - }, - "pinstripes/ps/minimax-m2.7": { - "max_tokens": 1000192, - "max_input_tokens": 1000192, - "max_output_tokens": 1000192, - "input_cost_per_token": 0.000000255, - "output_cost_per_token": 0.00000055, - "litellm_provider": "pinstripes", - "mode": "chat", - "supports_function_calling": true, - "supports_assistant_prefill": true, - "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" - }, - "fallback_generalizations": { - "rules": [ - { - "name": "anthropic-claude-adaptive-thinking", - "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", - "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", - "extends": "anthropic-claude", - "model_info": { - "supports_adaptive_thinking": true - } - }, - { - "name": "anthropic-claude", - "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", - "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", - "model_info": { - "litellm_provider": "anthropic", - "mode": "chat", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_tool_choice": true, - "supports_assistant_prefill": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_reasoning": true, - "supports_pdf_input": true, - "supports_system_messages": true - } - } - ] - } + "fallback_generalizations": { + "rules": [ + { + "name": "anthropic-claude-adaptive-thinking", + "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", + "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", + "extends": "anthropic-claude", + "model_info": { + "supports_adaptive_thinking": true + } + }, + { + "name": "anthropic-claude", + "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", + "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "model_info": { + "litellm_provider": "anthropic", + "mode": "chat", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_tool_choice": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_pdf_input": true, + "supports_system_messages": true + } + } + ] + } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 73cefeb7c77..15b16118a7b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1010,7 +1010,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1042,7 +1041,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1074,7 +1072,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1106,7 +1103,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1138,7 +1134,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1170,7 +1165,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1219,7 +1213,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1253,7 +1246,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1287,7 +1279,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1321,7 +1312,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1487,7 +1477,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1521,7 +1510,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1555,7 +1543,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1589,7 +1576,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1623,7 +1609,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1688,7 +1673,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1719,7 +1703,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1750,7 +1733,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1781,7 +1763,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1812,7 +1793,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1843,7 +1823,186 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "anthropic.claude-sonnet-5": { "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "global.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "us.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "eu.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "au.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_native_structured_output": true, + "supports_output_config": true + }, + "jp.anthropic.claude-sonnet-5": { + "supports_adaptive_thinking": true, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -2364,7 +2523,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2394,7 +2552,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2455,7 +2612,6 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -2523,6 +2679,34 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, + "azure_ai/claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, @@ -9132,17 +9316,16 @@ }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -9754,17 +9937,16 @@ }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "output_cost_per_token": 1.2e-06 + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -10274,6 +10456,35 @@ "supports_vision": true, "supports_output_config": true }, + "claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -19105,7 +19316,6 @@ "supported_endpoints": [ "/v1/chat/completions" ], - "supports_adaptive_thinking": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true @@ -21857,8 +22067,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21906,8 +22116,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21951,8 +22161,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21996,8 +22206,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22045,8 +22255,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22093,8 +22303,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22134,8 +22344,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22178,8 +22388,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22223,8 +22433,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22269,8 +22479,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22312,8 +22522,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22355,8 +22565,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -24873,14 +25083,13 @@ }, "minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.2e-06, "supports_function_calling": true, - "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -28336,7 +28545,6 @@ "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, "source": "https://openrouter.ai/anthropic/claude-sonnet-4.6", - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28376,7 +28584,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -28438,7 +28645,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -30377,7 +30583,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -30387,7 +30592,6 @@ "supports_adaptive_thinking": true, "litellm_provider": "perplexity", "mode": "responses", - "supports_adaptive_thinking": true, "supports_web_search": true, "supports_reasoning": false, "supports_function_calling": true, @@ -31434,15 +31638,15 @@ "supports_vision": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "snowflake/claude-3-5-sonnet": { + "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 200000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, "supports_computer_use": true, "supports_function_calling": true, "supports_vision": true, @@ -31450,14 +31654,14 @@ "supports_system_messages": true, "supports_response_schema": true }, - "snowflake/deepseek-r1": { + "snowflake/deepseek-r1": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000135, - "output_cost_per_token": 0.0000054, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 5.4e-06, "supports_reasoning": true, "supports_system_messages": true }, @@ -31516,8 +31720,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.0000012, - "output_cost_per_token": 0.0000012, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 1.2e-06, "supports_function_calling": true, "supports_system_messages": true }, @@ -31527,8 +31731,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true }, @@ -31538,8 +31742,8 @@ "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000024, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 2.4e-07, "supports_system_messages": true }, "snowflake/llama3.2-1b": { @@ -31556,17 +31760,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/llama3.3-70b": { + "snowflake/llama3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "snowflake/mistral-7b": { "litellm_provider": "snowflake", "max_input_tokens": 32000, @@ -31581,14 +31785,14 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/mistral-large2": { + "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_response_schema": true @@ -31628,17 +31832,17 @@ "max_tokens": 8192, "mode": "chat" }, - "snowflake/snowflake-llama-3.3-70b": { + "snowflake/snowflake-llama-3.3-70b": { "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, - "input_cost_per_token": 0.00000072, - "output_cost_per_token": 0.00000072, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, "litellm_provider": "snowflake", "mode": "chat", "supports_function_calling": true, "supports_system_messages": true - }, + }, "stability/sd3": { "litellm_provider": "stability", "mode": "image_generation", @@ -32890,7 +33094,7 @@ "input_cost_per_token": 3.6e-06, "input_cost_per_token_above_200k_tokens": 7.2e-06, "output_cost_per_token_above_200k_tokens": 2.7e-05, - "cache_creation_input_token_cost_above_200k_tokens": 9.0e-06, + "cache_creation_input_token_cost_above_200k_tokens": 9e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "litellm_provider": "bedrock_converse", @@ -33632,7 +33836,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -34867,7 +35070,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34897,7 +35099,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34927,7 +35128,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -34958,7 +35158,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35049,7 +35248,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35080,7 +35278,6 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -35133,7 +35330,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -37690,12 +37886,12 @@ }, "zai.glm-5": { "input_cost_per_token": 1e-06, + "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3.2e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -37716,20 +37912,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, @@ -42628,7 +42810,6 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_adaptive_thinking": true, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -42646,6 +42827,64 @@ }, "supports_output_config": true }, + "vertex_ai/claude-sonnet-5": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, + "vertex_ai/claude-sonnet-5@default": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_max_reasoning_effort": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_output_config": true + }, "duckduckgo/search": { "litellm_provider": "duckduckgo", "mode": "search", @@ -42662,7 +42901,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42677,7 +42919,10 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -42692,7 +42937,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42706,7 +42953,9 @@ "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "supported_endpoints": ["/v1/chat/completions"], + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42722,9 +42971,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42742,9 +42998,16 @@ "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/responses"], - "supported_modalities": ["text", "image"], - "supported_output_modalities": ["text"], + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -42761,7 +43024,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42777,7 +43043,10 @@ "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42793,7 +43062,10 @@ "max_tokens": 128000, "mode": "chat", "use_openai_responses_path": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_reasoning": true, @@ -42952,20 +43224,6 @@ } ] }, - "zai.glm-5": { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, @@ -42994,45 +43252,6 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, - "minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-east-1/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "bedrock/us-west-2/minimax.minimax-m2.5": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, "cache_creation_input_token_cost_above_1hr": 2.4e-06, @@ -43079,363 +43298,382 @@ "supports_native_structured_output": true, "supports_pdf_input": true }, - "snowflake/claude-sonnet-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-sonnet-4-6": { - "supports_adaptive_thinking": true, - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-4-opus": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000025, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/claude-haiku-4-5": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000005, - "cache_read_input_token_cost": 0.0000001, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/claude-3-7-sonnet": { - "max_tokens": 16384, - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-4.1": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000008, - "cache_read_input_token_cost": 0.0000005, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5": { - "max_tokens": 16384, - "max_input_tokens": 300000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000125, - "output_cost_per_token": 0.00001, - "cache_read_input_token_cost": 0.000000125, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-mini": { - "max_tokens": 16384, - "max_input_tokens": 1000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000012, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/openai-gpt-5-nano": { - "max_tokens": 16384, - "max_input_tokens": 5000000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.0000006, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true, - "supports_response_schema": true - }, - "snowflake/llama4-maverick": { - "max_tokens": 16384, - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "input_cost_per_token": 0.00000024, - "output_cost_per_token": 0.00000097, - "litellm_provider": "snowflake", - "mode": "chat", - "supports_function_calling": true, - "supports_system_messages": true - }, - "snowflake/snowflake-arctic-embed-l-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "snowflake/snowflake-arctic-embed-m-v2.0": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "input_cost_per_token": 0.00000007, - "output_cost_per_token": 0.0, - "litellm_provider": "snowflake", - "mode": "embedding" - }, - "soniox/stt-async-v4": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "soniox/stt-async-v5": { - "litellm_provider": "soniox", - "max_output_tokens": 8000, - "max_tokens": 8000, - "input_cost_per_second": 0.0, - "output_cost_per_second": 0.0000277778, - "mode": "audio_transcription", - "source": "https://soniox.com/pricing", - "supported_endpoints": ["/v1/audio/transcriptions"], - "supports_audio_input": true - }, - "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 6e-07, - "output_cost_per_token": 3.6e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 4.5e-07, - "output_cost_per_token": 1.8e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/Qwen/Qwen3.6-27B-FP8": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3.2e-07, - "output_cost_per_token": 3.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 202752, - "max_output_tokens": 202752, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/moonshotai/Kimi-K2.6": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 9.6e-07, - "output_cost_per_token": 4e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/MiniMaxAI/MiniMax-M2.5": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "cache_read_input_token_cost": 0, - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/google/gemma-4-31B-it": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.4e-07, - "output_cost_per_token": 5.6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-120b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 6e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - }, - "tensormesh/openai/gpt-oss-20b": { - "litellm_provider": "tensormesh", - "mode": "chat", - "input_cost_per_token": 7e-08, - "output_cost_per_token": 2.8e-07, - "cache_read_input_token_cost": 0, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_reasoning": true, - "source": "https://serverless.tensormesh.ai/v1/models/openrouter" - } - , + "snowflake/claude-sonnet-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-sonnet-4-6": { + "supports_adaptive_thinking": true, + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-sonnet-5": { + "supports_adaptive_thinking": true, + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-4-opus": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/claude-haiku-4-5": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/claude-3-7-sonnet": { + "max_tokens": 16384, + "max_input_tokens": 200000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-4.1": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5": { + "max_tokens": 16384, + "max_input_tokens": 300000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-mini": { + "max_tokens": 16384, + "max_input_tokens": 1000000, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/openai-gpt-5-nano": { + "max_tokens": 16384, + "max_input_tokens": 5000000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "snowflake/llama4-maverick": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9.7e-07, + "litellm_provider": "snowflake", + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true + }, + "snowflake/snowflake-arctic-embed-l-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "snowflake/snowflake-arctic-embed-m-v2.0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "snowflake", + "mode": "embedding" + }, + "soniox/stt-async-v4": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "soniox/stt-async-v5": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true + }, + "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 6e-07, + "output_cost_per_token": 3.6e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 4.5e-07, + "output_cost_per_token": 1.8e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/Qwen/Qwen3.6-27B-FP8": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 3.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 202752, + "max_output_tokens": 202752, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/deepseek-ai/DeepSeek-V4-Flash": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/moonshotai/Kimi-K2.6": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 9.6e-07, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/MiniMaxAI/MiniMax-M2.5": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 0, + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/google/gemma-4-31B-it": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 5.6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-120b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, + "tensormesh/openai/gpt-oss-20b": { + "litellm_provider": "tensormesh", + "mode": "chat", + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 0, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_reasoning": true, + "source": "https://serverless.tensormesh.ai/v1/models/openrouter" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 2.8e-09, @@ -43540,8 +43778,8 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.000000125, - "output_cost_per_token": 0.00000045, + "input_cost_per_token": 1.25e-07, + "output_cost_per_token": 4.5e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43553,8 +43791,8 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.00000014, - "output_cost_per_token": 0.00000045, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 4.5e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43566,8 +43804,8 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.00000009, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 2e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43579,8 +43817,8 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.0000003, - "output_cost_per_token": 0.0000006, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43592,8 +43830,8 @@ "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000002, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43605,8 +43843,8 @@ "max_tokens": 1000192, "max_input_tokens": 1000192, "max_output_tokens": 1000192, - "input_cost_per_token": 0.000000255, - "output_cost_per_token": 0.00000055, + "input_cost_per_token": 2.55e-07, + "output_cost_per_token": 5.5e-07, "litellm_provider": "pinstripes", "mode": "chat", "supports_function_calling": true, @@ -43648,39 +43886,39 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "fallback_generalizations": { - "rules": [ - { - "name": "anthropic-claude-adaptive-thinking", - "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", - "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", - "extends": "anthropic-claude", - "model_info": { - "supports_adaptive_thinking": true - } - }, - { - "name": "anthropic-claude", - "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", - "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", - "model_info": { - "litellm_provider": "anthropic", - "mode": "chat", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_tool_choice": true, - "supports_assistant_prefill": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_reasoning": true, - "supports_pdf_input": true, - "supports_system_messages": true - } - } - ] - } + "fallback_generalizations": { + "rules": [ + { + "name": "anthropic-claude-adaptive-thinking", + "pattern": "(?:opus|sonnet|haiku)[-._](?:4[-._](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d{1,})[-._]\\d{1,2}(?!\\d))", + "description": "Claude opus/sonnet/haiku at version 4.6 or higher: 4.6 through 4.99, then any 5.x, 6.x or later major. The minor is capped at two digits so an 8-digit date suffix such as claude-opus-4-20250514 is never read as a >= 4.6 minor. Turns on adaptive thinking for new families with no code change.", + "extends": "anthropic-claude", + "model_info": { + "supports_adaptive_thinking": true + } + }, + { + "name": "anthropic-claude", + "pattern": "^claude-[a-z]+-\\d+[-.]\\d+(?:-\\d{8})?$", + "description": "Any Claude family-major-minor id, optionally with an 8-digit date suffix, anchored to the whole name. Version-neutral fallback that gives an unmapped Claude provider routing and baseline capabilities; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "model_info": { + "litellm_provider": "anthropic", + "mode": "chat", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_tool_choice": true, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_pdf_input": true, + "supports_system_messages": true + } + } + ] + } } diff --git a/tests/test_litellm/test_claude_sonnet_5_config.py b/tests/test_litellm/test_claude_sonnet_5_config.py new file mode 100644 index 00000000000..943c4172205 --- /dev/null +++ b/tests/test_litellm/test_claude_sonnet_5_config.py @@ -0,0 +1,163 @@ +""" +Validate Claude Sonnet 5 model configuration entries. + +Sonnet 5 launched 2026-06-29 with introductory pricing ($2/$10 per MTok) +through August 31, 2026, after which it moves to standard pricing ($3/$15). +The cost-map entries use the introductory rates so spend tracking is accurate +during the promotional period. +""" + +import json +import os + +import pytest + +import litellm +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + +REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") + + +def _load_root_cost_map() -> dict: + json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json") + with open(json_path) as f: + return json.load(f) + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + original_model_cost = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + + +def test_sonnet_5_model_pricing_and_capabilities(): + model_data = _load_root_cost_map() + + expected_models = [ + ("claude-sonnet-5", "anthropic"), + ("anthropic.claude-sonnet-5", "bedrock_converse"), + ("vertex_ai/claude-sonnet-5", "vertex_ai-anthropic_models"), + ("azure_ai/claude-sonnet-5", "azure_ai"), + ] + + for model_name, provider in expected_models: + assert model_name in model_data, f"Missing model entry: {model_name}" + info = model_data[model_name] + + assert info["litellm_provider"] == provider + assert info["mode"] == "chat" + assert info["max_input_tokens"] == 1000000 + assert info["max_output_tokens"] == 128000 + assert info["max_tokens"] == 128000 + + assert info["input_cost_per_token"] == 2e-06 + assert info["output_cost_per_token"] == 1e-05 + assert info["cache_creation_input_token_cost"] == 2.5e-06 + assert info["cache_creation_input_token_cost_above_1hr"] == 4e-06 + assert info["cache_read_input_token_cost"] == 2e-07 + + assert "input_cost_per_token_above_200k_tokens" not in info + assert "output_cost_per_token_above_200k_tokens" not in info + + assert info["supports_adaptive_thinking"] is True + assert info["supports_function_calling"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_reasoning"] is True + assert info["supports_tool_choice"] is True + assert info["supports_vision"] is True + + +def test_sonnet_5_bedrock_regional_model_pricing(): + model_data = _load_root_cost_map() + + expected_models = { + "global.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_creation_input_token_cost": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + }, + "us.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05, + "cache_creation_input_token_cost": 2.75e-06, + "cache_read_input_token_cost": 2.2e-07, + }, + "eu.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05, + "cache_creation_input_token_cost": 2.75e-06, + "cache_read_input_token_cost": 2.2e-07, + }, + "au.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05, + "cache_creation_input_token_cost": 2.75e-06, + "cache_read_input_token_cost": 2.2e-07, + }, + "jp.anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05, + "cache_creation_input_token_cost": 2.75e-06, + "cache_read_input_token_cost": 2.2e-07, + }, + } + + for model_name, expected in expected_models.items(): + assert model_name in model_data, f"Missing model entry: {model_name}" + info = model_data[model_name] + assert info["litellm_provider"] == "bedrock_converse" + assert info["max_input_tokens"] == 1000000 + assert info["max_output_tokens"] == 128000 + for key, value in expected.items(): + assert info[key] == value, f"{model_name}.{key}: expected {value}, got {info[key]}" + + +def test_sonnet_5_present_in_bundled_backup(): + backup = GetModelCostMap.load_local_model_cost_map() + root = _load_root_cost_map() + for model_name in ( + "claude-sonnet-5", + "anthropic.claude-sonnet-5", + "global.anthropic.claude-sonnet-5", + "us.anthropic.claude-sonnet-5", + "eu.anthropic.claude-sonnet-5", + "au.anthropic.claude-sonnet-5", + "jp.anthropic.claude-sonnet-5", + "vertex_ai/claude-sonnet-5", + "vertex_ai/claude-sonnet-5@default", + "azure_ai/claude-sonnet-5", + "snowflake/claude-sonnet-5", + ): + assert model_name in backup, f"Missing from backup cost map: {model_name}" + assert backup[model_name] == root[model_name], model_name + + +def test_sonnet_5_provider_resolves_via_model_info(local_model_cost_map): + info = litellm.get_model_info(model="claude-sonnet-5") + assert info["litellm_provider"] == "anthropic" + assert info["max_input_tokens"] == 1000000 + assert info["max_output_tokens"] == 128000 + assert info["input_cost_per_token"] == 2e-06 + assert info["output_cost_per_token"] == 1e-05 + + +@pytest.mark.parametrize( + "cost_map", + [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], + ids=["root", "bundled_backup"], +) +def test_sonnet_5_all_variants_carry_adaptive_thinking_flag(cost_map): + variants = [k for k in cost_map if "claude-sonnet-5" in k and "sonnet-4-5" not in k] + assert variants, "no claude-sonnet-5 entries found in cost map" + missing = [ + k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True + ] + assert not missing, f"missing supports_adaptive_thinking: {missing}" From 901a097b86ddb9539982173bcc91a6a29a27a2d7 Mon Sep 17 00:00:00 2001 From: unknown <> Date: Wed, 1 Jul 2026 14:35:36 +0000 Subject: [PATCH 007/411] fix: restore supports_reasoning on minimax entries lost during dedup The original JSON had duplicate minimax.minimax-m2.5 entries; the first had supports_reasoning: true, the second did not. JSON deduplication kept the second. Restore the flag on the surviving entries and add 1hr cache TTL assertions to the regional bedrock test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 9 ++++++--- model_prices_and_context_window.json | 9 ++++++--- tests/test_litellm/test_claude_sonnet_5_config.py | 5 +++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 15b16118a7b..dba8bb949d0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9325,7 +9325,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -9946,7 +9947,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -25092,7 +25094,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 15b16118a7b..dba8bb949d0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9325,7 +9325,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -9946,7 +9947,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, @@ -25092,7 +25094,8 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_reasoning": true }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, diff --git a/tests/test_litellm/test_claude_sonnet_5_config.py b/tests/test_litellm/test_claude_sonnet_5_config.py index 943c4172205..c8726a039f2 100644 --- a/tests/test_litellm/test_claude_sonnet_5_config.py +++ b/tests/test_litellm/test_claude_sonnet_5_config.py @@ -82,30 +82,35 @@ def test_sonnet_5_bedrock_regional_model_pricing(): "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, }, "us.anthropic.claude-sonnet-5": { "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, }, "eu.anthropic.claude-sonnet-5": { "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, }, "au.anthropic.claude-sonnet-5": { "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, }, "jp.anthropic.claude-sonnet-5": { "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, }, } From f06fdab04bd5001f76604636d4edf0d9a5766d11 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:11:16 +0000 Subject: [PATCH 008/411] fix(bedrock): stop leaking thinking/reasoning_effort into additionalModelRequestFields for DeepSeek --- .../bedrock/chat/converse_transformation.py | 29 +++++++---- .../chat/test_converse_transformation.py | 52 +++++++++++++++++++ 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index c38b3593465..08f44d5b1b1 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -493,6 +493,23 @@ class AmazonConverseConfig(BaseConfig): ) thinking["budget_tokens"] = BEDROCK_MIN_THINKING_BUDGET_TOKENS + def _model_accepts_anthropic_thinking_param(self, model: str, base_model: str) -> bool: + """Whether the model accepts the Anthropic-shaped ``thinking`` / ``reasoning_effort`` request field. + + The Converse mapping serializes ``thinking`` into ``additionalModelRequestFields`` in Anthropic's + shape. Only Anthropic Claude reasoning models accept that field; DeepSeek models reason natively + and reject it (a 400 when it leaks through), even though they advertise ``supports_reasoning``. + """ + if "deepseek" in model or "deepseek" in base_model: + return False + return ( + "claude-3-7" in model + or "claude-sonnet-4" in model + or "claude-opus-4" in model + or supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider) + or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) + ) + def get_supported_openai_params(self, model: str) -> List[str]: from litellm.utils import supports_function_calling @@ -553,17 +570,7 @@ class AmazonConverseConfig(BaseConfig): # Nova 2 models support reasoning_effort (transformed to reasoningConfig) # These models use a different reasoning structure than Anthropic's thinking parameter supported_params.append("reasoning_effort") - elif ( - "claude-3-7" in model - or "claude-sonnet-4" in model - or "claude-opus-4" in model - or "deepseek.r1" in model - or supports_reasoning( - model=model, - custom_llm_provider=self.custom_llm_provider, - ) - or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) - ): + elif self._model_accepts_anthropic_thinking_param(model=model, base_model=base_model): supported_params.append("thinking") supported_params.append("reasoning_effort") diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index fc12ead36a1..f7e2716bd19 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -553,6 +553,58 @@ def test_get_supported_openai_params(): assert "reasoning_effort" in supported_params +@pytest.mark.parametrize( + "model", + [ + "bedrock/us.deepseek.r1-v1:0", + "bedrock/converse/us.deepseek.r1-v1:0", + "bedrock/deepseek.v3-v1:0", + "bedrock/deepseek.v3.2", + ], +) +def test_bedrock_deepseek_does_not_advertise_thinking(model): + """DeepSeek reasons natively on Bedrock and rejects the Anthropic-shaped + `thinking`/`reasoning_effort` field, so it must not be advertised as supported + (otherwise it leaks into additionalModelRequestFields and Bedrock 400s).""" + config = AmazonConverseConfig() + supported_params = config.get_supported_openai_params(model=model) + assert "thinking" not in supported_params + assert "reasoning_effort" not in supported_params + + +def test_bedrock_deepseek_r1_thinking_raises_without_drop_params(): + """Passing `thinking` to Bedrock DeepSeek R1 must fail client-side with a clear + UnsupportedParamsError instead of leaking through and hitting a Bedrock 400.""" + with pytest.raises(litellm.UnsupportedParamsError): + litellm.utils.get_optional_params( + model="us.deepseek.r1-v1:0", + custom_llm_provider="bedrock", + thinking={"type": "enabled", "budget_tokens": 1024}, + ) + + +def test_bedrock_deepseek_r1_thinking_dropped_does_not_leak_into_request(): + """With drop_params, `thinking` is dropped rather than forwarded into + additionalModelRequestFields for Bedrock DeepSeek R1.""" + optional_params = litellm.utils.get_optional_params( + model="us.deepseek.r1-v1:0", + custom_llm_provider="bedrock", + thinking={"type": "enabled", "budget_tokens": 1024}, + drop_params=True, + ) + assert "thinking" not in optional_params + + config = AmazonConverseConfig() + request = config._transform_request( + model="bedrock/converse/us.deepseek.r1-v1:0", + messages=[{"role": "user", "content": "Say hi in one word."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert "thinking" not in request.get("additionalModelRequestFields", {}) + + def test_get_supported_openai_params_bedrock_converse(): """ Test that all documented bedrock converse models have the same set of supported openai params when using From f4ad6b0f0bd13bad1fd8ee68e6b5463730b50e0e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:20:03 +0000 Subject: [PATCH 009/411] fix(bedrock): drop thinking/reasoning_effort in converse mapping for DeepSeek --- .../bedrock/chat/converse_transformation.py | 15 +++++++++- .../chat/test_converse_transformation.py | 28 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 08f44d5b1b1..d7ee15cedd7 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -857,6 +857,9 @@ class AmazonConverseConfig(BaseConfig): drop_params: bool, ) -> dict: is_thinking_enabled = self.is_thinking_enabled(non_default_params) + accepts_thinking_param = self._model_accepts_anthropic_thinking_param( + model=model, base_model=BedrockModelInfo.get_base_model(model) + ) for param, value in non_default_params.items(): if param == "response_format" and isinstance(value, dict): @@ -906,7 +909,12 @@ class AmazonConverseConfig(BaseConfig): optional_params["_parallel_tool_use_config"] = { "tool_choice": {"disable_parallel_tool_use": disable_parallel} } - if param == "thinking": + if param == "thinking" and not accepts_thinking_param: + verbose_logger.debug( + "Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively and rejects it.", + model, + ) + elif param == "thinking": if ( isinstance(value, dict) and value.get("type") == "adaptive" @@ -929,6 +937,11 @@ class AmazonConverseConfig(BaseConfig): litellm.verbose_logger.warning(DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, model) else: optional_params["thinking"] = value + elif param == "reasoning_effort" and isinstance(value, str) and not accepts_thinking_param: + verbose_logger.debug( + "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it reasons natively and rejects it.", + model, + ) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index f7e2716bd19..93455f00143 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -605,6 +605,34 @@ def test_bedrock_deepseek_r1_thinking_dropped_does_not_leak_into_request(): assert "thinking" not in request.get("additionalModelRequestFields", {}) +@pytest.mark.parametrize("param", ["thinking", "reasoning_effort"]) +def test_bedrock_deepseek_r1_reasoning_params_not_forwarded_by_map(param): + """Even when map_openai_params is called directly (bypassing the supported-params + gate), DeepSeek R1 must not forward the Anthropic-shaped thinking/reasoning_effort + into additionalModelRequestFields, since Bedrock rejects it with a 400.""" + config = AmazonConverseConfig() + model = "bedrock/converse/us.deepseek.r1-v1:0" + value = {"type": "enabled", "budget_tokens": 1024} if param == "thinking" else "high" + + optional_params = config.map_openai_params( + non_default_params={param: value, "max_tokens": 100}, + optional_params={}, + model=model, + drop_params=False, + ) + assert "thinking" not in optional_params + assert "reasoning_effort" not in optional_params + + request = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Say hi in one word."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert request.get("additionalModelRequestFields") is None + + def test_get_supported_openai_params_bedrock_converse(): """ Test that all documented bedrock converse models have the same set of supported openai params when using From ab1f739188cf4096ae7882a6b7848e702480ed02 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 23 Jul 2026 03:29:45 +0000 Subject: [PATCH 010/411] fix(bedrock): only drop reasoning params for DeepSeek, not all non-allowlisted models The DeepSeek leak fix gated the drop of thinking/reasoning_effort on a positive allowlist (_model_accepts_anthropic_thinking_param). Its negation dropped the params for any model the allowlist and ARN introspection missed, so Claude behind application-inference-profile ARNs, gpt-oss-safeguard (absent from the cost map so supports_reasoning is False), and Nova 2 custom-import ARNs all silently lost reasoning in prod. Gate the drop on a narrow denylist instead: only DeepSeek reasons natively and 400s on the request field, so drop only there and leave every other model (including opaque ARNs and gpt-oss/Nova 2 which route reasoning_effort through their own shapes) untouched. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock/chat/converse_transformation.py | 18 ++++++-- .../chat/test_converse_transformation.py | 46 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d7ee15cedd7..617ce423d51 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -510,6 +510,18 @@ class AmazonConverseConfig(BaseConfig): or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) ) + def _model_reasons_natively_and_rejects_request_param(self, model: str, base_model: str) -> bool: + """Whether the model reasons natively and rejects any reasoning request field on Converse. + + The Converse mapping serializes ``thinking`` into ``additionalModelRequestFields`` in Anthropic's + shape and ``reasoning_effort`` into a provider-specific shape. DeepSeek reasons on its own and + returns a 400 when either field is sent, even though it advertises ``supports_reasoning``, so both + must be dropped for it. Every other model either accepts one of those shapes (Claude ``thinking``, + gpt-oss / Nova 2 ``reasoning_effort``) or is an opaque ARN we can't introspect, so we leave those + untouched rather than silently degrading reasoning. + """ + return "deepseek" in model or "deepseek" in base_model + def get_supported_openai_params(self, model: str) -> List[str]: from litellm.utils import supports_function_calling @@ -857,7 +869,7 @@ class AmazonConverseConfig(BaseConfig): drop_params: bool, ) -> dict: is_thinking_enabled = self.is_thinking_enabled(non_default_params) - accepts_thinking_param = self._model_accepts_anthropic_thinking_param( + drop_reasoning_request_param = self._model_reasons_natively_and_rejects_request_param( model=model, base_model=BedrockModelInfo.get_base_model(model) ) @@ -909,7 +921,7 @@ class AmazonConverseConfig(BaseConfig): optional_params["_parallel_tool_use_config"] = { "tool_choice": {"disable_parallel_tool_use": disable_parallel} } - if param == "thinking" and not accepts_thinking_param: + if param == "thinking" and drop_reasoning_request_param: verbose_logger.debug( "Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively and rejects it.", model, @@ -937,7 +949,7 @@ class AmazonConverseConfig(BaseConfig): litellm.verbose_logger.warning(DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, model) else: optional_params["thinking"] = value - elif param == "reasoning_effort" and isinstance(value, str) and not accepts_thinking_param: + elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_request_param: verbose_logger.debug( "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it reasons natively and rejects it.", model, diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 93455f00143..87a172f2409 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -633,6 +633,52 @@ def test_bedrock_deepseek_r1_reasoning_params_not_forwarded_by_map(param): assert request.get("additionalModelRequestFields") is None +@pytest.mark.parametrize( + "model, param, value, kept_key", + [ + ( + "bedrock/us.anthropic.claude-opus-4-20250514-v1:0", + "thinking", + {"type": "enabled", "budget_tokens": 1024}, + "thinking", + ), + ( + "bedrock/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", + "thinking", + {"type": "enabled", "budget_tokens": 1024}, + "thinking", + ), + ( + "bedrock/openai.gpt-oss-safeguard-20b-1:0", + "reasoning_effort", + "high", + "reasoning_effort", + ), + ( + "bedrock/us.amazon.nova-2-lite-v1:0", + "reasoning_effort", + "high", + "reasoningConfig", + ), + ], +) +def test_bedrock_non_deepseek_reasoning_params_preserved(model, param, value, kept_key): + """The DeepSeek leak fix must only drop reasoning request params for DeepSeek. + + Claude behind an application-inference-profile ARN, gpt-oss-safeguard (absent from the + cost map so `supports_reasoning` is False), and Nova 2 all reason via a request param and + must keep it. Regression guard against gating the drop on a positive allowlist, which + silently degraded reasoning for anything the allowlist/ARN introspection missed.""" + config = AmazonConverseConfig() + optional_params = config.map_openai_params( + non_default_params={param: value, "max_tokens": 100}, + optional_params={}, + model=model, + drop_params=False, + ) + assert kept_key in optional_params + + def test_get_supported_openai_params_bedrock_converse(): """ Test that all documented bedrock converse models have the same set of supported openai params when using From f88dd65590460df01c49a6a58ab95ee7f28d8b58 Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 14:31:25 +0200 Subject: [PATCH 011/411] feat(voyage): add voyage-4 family + voyage-context-4, fix contextual list[str] input - Add voyage-4, voyage-4-large, voyage-4-lite, voyage-4-nano (/bin/bash) and voyage-context-4 to the cost/context map (both root and backup json). - Contextual embeddings: normalize flat list[str] input. Voyage rejects a flat list[str] unless input_type='query', so wrap each string as its own single-chunk document otherwise; single str -> [[str]]; list[list[str]] passes through. Keeps list[str] when input_type='query'. - Add tests for input normalization and voyage-4 family pricing. --- .../embedding/transformation_contextual.py | 33 ++++++- ...odel_prices_and_context_window_backup.json | 40 +++++++++ model_prices_and_context_window.json | 40 +++++++++ tests/llm_translation/test_voyage_ai.py | 85 +++++++++++++++++++ 4 files changed, 197 insertions(+), 1 deletion(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 4df2fa4ba31..2d661610eaa 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -106,11 +106,42 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): headers: dict, ) -> dict: return { - "inputs": input, + "inputs": self._transform_contextual_inputs(input, optional_params), "model": model, **optional_params, } + @staticmethod + def _transform_contextual_inputs( + input: Union[AllEmbeddingInputValues, List[List[str]]], + optional_params: dict, + ) -> List[List[str]]: + """ + Voyage's contextualized embeddings API expects ``inputs`` to be a + ``list[list[str]]`` (each inner list is a document made of chunks that + share context). + + It also accepts a flat ``list[str]`` - but only when + ``input_type == "query"``. In every other case a flat list must be + wrapped so each string becomes its own single-chunk document, otherwise + the API rejects the request with a 400. + + Reference: https://docs.voyageai.com/reference/contextualized-embeddings-api + """ + # Single string -> one document with a single chunk. + if isinstance(input, str): + return [[input]] + + # Flat list[str]: keep as list[str] when the API allows it + # (input_type="query"), otherwise wrap each string as its own document. + if isinstance(input, list) and all(isinstance(i, str) for i in input): + if optional_params.get("input_type") == "query": + return input # type: ignore[return-value] + return [[i] for i in input] + + # Already list[list[str]] (or another shape) -> pass through unchanged. + return input # type: ignore[return-value] + def transform_embedding_response( self, model: str, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c584deb683a..fc710497592 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27345,6 +27345,38 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-4": { + "input_cost_per_token": 6e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-large": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-nano": { + "input_cost_per_token": 0.0, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, "voyage/voyage-code-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", @@ -27369,6 +27401,14 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-context-4": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 120000, + "max_tokens": 120000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, "voyage/voyage-finance-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c584deb683a..fc710497592 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27345,6 +27345,38 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-4": { + "input_cost_per_token": 6e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-large": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "voyage/voyage-4-nano": { + "input_cost_per_token": 0.0, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, "voyage/voyage-code-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", @@ -27369,6 +27401,14 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-context-4": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 120000, + "max_tokens": 120000, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, "voyage/voyage-finance-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", diff --git a/tests/llm_translation/test_voyage_ai.py b/tests/llm_translation/test_voyage_ai.py index a0b9ee0a44b..577ee808035 100644 --- a/tests/llm_translation/test_voyage_ai.py +++ b/tests/llm_translation/test_voyage_ai.py @@ -71,6 +71,26 @@ class TestVoyageAI(BaseLLMEmbeddingTest): assert response.usage.total_tokens > 0 +@pytest.mark.parametrize( + "model, expected_input_cost", + [ + ("voyage/voyage-4", 6e-08), + ("voyage/voyage-4-large", 1.2e-07), + ("voyage/voyage-4-lite", 2e-08), + ("voyage/voyage-4-nano", 0.0), + ("voyage/voyage-context-4", 1.2e-07), + ], +) +def test_voyage_4_family_pricing_registered(model, expected_input_cost): + """The voyage-4 family and voyage-context-4 must be in the cost map.""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + info = litellm.get_model_info(model=model) + assert info["litellm_provider"] == "voyage" + assert info["mode"] == "embedding" + assert info["input_cost_per_token"] == expected_input_cost + + def test_voyage_ai_embedding_extra_params(): """Test Voyage AI embedding with extra parameters""" try: @@ -142,6 +162,7 @@ class TestVoyageContextualEmbeddings: # Test contextual model detection assert config.is_contextualized_embeddings("voyage-context-3") is True + assert config.is_contextualized_embeddings("voyage-context-4") is True assert config.is_contextualized_embeddings("voyage-context-2") is True assert config.is_contextualized_embeddings("context-model") is True @@ -198,6 +219,70 @@ class TestVoyageContextualEmbeddings: assert transformed["model"] == "voyage-context-3" assert transformed["encoding_format"] == "float" + def test_contextual_flat_list_str_is_wrapped(self): + """Flat list[str] without input_type must be wrapped to list[list[str]].""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", ["Hello", "world"], {}, {} + ) + + # Voyage rejects a flat list[str] unless input_type="query", so each + # string becomes its own single-chunk document. + assert transformed["inputs"] == [["Hello"], ["world"]] + assert transformed["model"] == "voyage-context-4" + + def test_contextual_flat_list_str_query_stays_flat(self): + """Flat list[str] with input_type='query' is sent as-is (list[str]).""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", + ["Hello", "world"], + {"input_type": "query"}, + {}, + ) + + assert transformed["inputs"] == ["Hello", "world"] + assert transformed["input_type"] == "query" + + def test_contextual_single_string_is_wrapped(self): + """A single string is wrapped to a single document with a single chunk.""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", "Hello", {}, {} + ) + + assert transformed["inputs"] == [["Hello"]] + + def test_contextual_nested_input_passthrough(self): + """Already-nested list[list[str]] input is passed through unchanged.""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + nested = [["Hello", "world"], ["Test"]] + transformed = config.transform_embedding_request( + "voyage-context-4", nested, {}, {} + ) + + assert transformed["inputs"] == nested + def test_contextual_embedding_response_transformation(self): """Test response transformation for contextual embeddings""" from litellm.llms.voyage.embedding.transformation_contextual import ( From 328b54d0042d213aee3e1e2c025480eae5a214d8 Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 15:22:31 +0200 Subject: [PATCH 012/411] fix(voyage): auto-chunk flat list[str] for contextual embeddings; fix mypy Rework per review: - Contextual inputs now prefer a flat list[str] and let the API auto-chunk (enable_auto_chunking=True, chunk_size=32000, input_type=document) instead of pre-wrapping into list[list[str]]. Matches the live API contract: * flat list[str] + input_type=query -> kept flat * flat list[str] otherwise / single str -> flat + auto-chunking * list[list[str]] -> passthrough - Fix mypy list-item error from the previous wrapping approach. - Update tests for the new normalization; all pass. --- .../embedding/transformation_contextual.py | 73 ++++++++++++++----- tests/llm_translation/test_voyage_ai.py | 47 ++++++++++-- 2 files changed, 94 insertions(+), 26 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 2d661610eaa..d555a1a2266 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -2,7 +2,7 @@ This module is used to transform the request and response for the Voyage contextualized embeddings API. This would be used for all the contextualized embeddings models in Voyage. """ -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Tuple, Union import httpx @@ -98,6 +98,11 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): "Authorization": f"Bearer {api_key}", } + # Chunk size (in tokens) used when the API auto-chunks a flat ``list[str]``. + # Matches the voyage-context-4 context window so each string stays a single + # chunk instead of being split. + AUTO_CHUNK_SIZE = 32000 + def transform_embedding_request( self, model: str, @@ -105,42 +110,74 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict, headers: dict, ) -> dict: + inputs, extra_params = self._transform_contextual_inputs(input, optional_params) return { - "inputs": self._transform_contextual_inputs(input, optional_params), + "inputs": inputs, "model": model, **optional_params, + **extra_params, } - @staticmethod + @classmethod def _transform_contextual_inputs( + cls, input: Union[AllEmbeddingInputValues, List[List[str]]], optional_params: dict, - ) -> List[List[str]]: + ) -> Tuple[Union[List[str], List[List[str]]], dict]: """ - Voyage's contextualized embeddings API expects ``inputs`` to be a - ``list[list[str]]`` (each inner list is a document made of chunks that - share context). + Normalize ``input`` for Voyage's contextualized embeddings API and + return ``(inputs, extra_params)`` where ``extra_params`` carries any + request fields (e.g. auto-chunking) needed for the chosen shape. - It also accepts a flat ``list[str]`` - but only when - ``input_type == "query"``. In every other case a flat list must be - wrapped so each string becomes its own single-chunk document, otherwise - the API rejects the request with a 400. + The API contract (verified against the live endpoint) is: + + - A flat ``list[str]`` is only accepted with ``input_type="query"`` or + with ``enable_auto_chunking=True`` (which itself requires + ``input_type="document"``). + - A ``list[list[str]]`` (each inner list = one document's chunks) is + always accepted. + + So we prefer to send a flat ``list[str]`` and let the API auto-chunk, + instead of pre-wrapping into ``list[list[str]]``: + + - ``str`` -> ``[str]`` + ``enable_auto_chunking`` (input_type=document) + - flat ``list[str]`` + ``input_type="query"`` -> kept flat, as-is + - flat ``list[str]`` otherwise -> kept flat + ``enable_auto_chunking`` + (input_type=document) + - ``list[list[str]]`` -> passed through unchanged Reference: https://docs.voyageai.com/reference/contextualized-embeddings-api """ - # Single string -> one document with a single chunk. + # Single string -> a one-element flat list, auto-chunked. if isinstance(input, str): - return [[input]] + return [input], cls._auto_chunk_params(optional_params) - # Flat list[str]: keep as list[str] when the API allows it - # (input_type="query"), otherwise wrap each string as its own document. + # Flat list[str]. if isinstance(input, list) and all(isinstance(i, str) for i in input): if optional_params.get("input_type") == "query": - return input # type: ignore[return-value] - return [[i] for i in input] + # The API accepts a flat query list as-is. + return input, {} # type: ignore[return-value] + # Otherwise let the API auto-chunk the flat list. + return input, cls._auto_chunk_params(optional_params) # type: ignore[return-value] # Already list[list[str]] (or another shape) -> pass through unchanged. - return input # type: ignore[return-value] + return input, {} # type: ignore[return-value] + + @classmethod + def _auto_chunk_params(cls, optional_params: dict) -> dict: + """ + Params required to send a flat ``list[str]`` to the contextualized API. + + ``enable_auto_chunking=True`` requires ``input_type="document"``, so set + it unless the caller already provided an ``input_type``. + """ + params: Dict[str, Any] = { + "enable_auto_chunking": True, + "chunk_size": cls.AUTO_CHUNK_SIZE, + } + if not optional_params.get("input_type"): + params["input_type"] = "document" + return params def transform_embedding_response( self, diff --git a/tests/llm_translation/test_voyage_ai.py b/tests/llm_translation/test_voyage_ai.py index 577ee808035..b7a2c41a979 100644 --- a/tests/llm_translation/test_voyage_ai.py +++ b/tests/llm_translation/test_voyage_ai.py @@ -219,8 +219,13 @@ class TestVoyageContextualEmbeddings: assert transformed["model"] == "voyage-context-3" assert transformed["encoding_format"] == "float" - def test_contextual_flat_list_str_is_wrapped(self): - """Flat list[str] without input_type must be wrapped to list[list[str]].""" + def test_contextual_flat_list_str_is_auto_chunked(self): + """Flat list[str] without input_type stays flat and is auto-chunked. + + The API accepts a flat list[str] only with input_type="query" or with + enable_auto_chunking=True (which requires input_type="document"), so we + keep the list flat and let the API auto-chunk it. + """ from litellm.llms.voyage.embedding.transformation_contextual import ( VoyageContextualEmbeddingConfig, ) @@ -231,10 +236,11 @@ class TestVoyageContextualEmbeddings: "voyage-context-4", ["Hello", "world"], {}, {} ) - # Voyage rejects a flat list[str] unless input_type="query", so each - # string becomes its own single-chunk document. - assert transformed["inputs"] == [["Hello"], ["world"]] + assert transformed["inputs"] == ["Hello", "world"] assert transformed["model"] == "voyage-context-4" + assert transformed["enable_auto_chunking"] is True + assert transformed["chunk_size"] == 32000 + assert transformed["input_type"] == "document" def test_contextual_flat_list_str_query_stays_flat(self): """Flat list[str] with input_type='query' is sent as-is (list[str]).""" @@ -253,9 +259,31 @@ class TestVoyageContextualEmbeddings: assert transformed["inputs"] == ["Hello", "world"] assert transformed["input_type"] == "query" + # A query list is accepted as-is, no auto-chunking needed. + assert "enable_auto_chunking" not in transformed - def test_contextual_single_string_is_wrapped(self): - """A single string is wrapped to a single document with a single chunk.""" + def test_contextual_flat_list_str_document_input_type_preserved(self): + """Explicit input_type='document' is preserved while auto-chunking.""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", + ["Hello", "world"], + {"input_type": "document"}, + {}, + ) + + assert transformed["inputs"] == ["Hello", "world"] + assert transformed["input_type"] == "document" + assert transformed["enable_auto_chunking"] is True + assert transformed["chunk_size"] == 32000 + + def test_contextual_single_string_is_auto_chunked(self): + """A single string becomes a one-element flat list and is auto-chunked.""" from litellm.llms.voyage.embedding.transformation_contextual import ( VoyageContextualEmbeddingConfig, ) @@ -266,7 +294,10 @@ class TestVoyageContextualEmbeddings: "voyage-context-4", "Hello", {}, {} ) - assert transformed["inputs"] == [["Hello"]] + assert transformed["inputs"] == ["Hello"] + assert transformed["enable_auto_chunking"] is True + assert transformed["chunk_size"] == 32000 + assert transformed["input_type"] == "document" def test_contextual_nested_input_passthrough(self): """Already-nested list[list[str]] input is passed through unchanged.""" From be6487d33c43dcf28de410fe209ede88712513b8 Mon Sep 17 00:00:00 2001 From: fzowl <160063452+fzowl@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:20:27 +0200 Subject: [PATCH 013/411] Update litellm/llms/voyage/embedding/transformation_contextual.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/llms/voyage/embedding/transformation_contextual.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 84ed856e080..c4eb6f34283 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -148,6 +148,8 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): """ # Single string -> a one-element flat list, auto-chunked. if isinstance(input, str): + if optional_params.get("input_type") == "query": + return [input], {} return [input], cls._auto_chunk_params(optional_params) # Flat list[str]. From e8ae29f135d35bbea296d4fa935ef1820c65bb0d Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 17:45:08 +0200 Subject: [PATCH 014/411] fix(voyage): ruff format + add missing query-string test coverage --- .../embedding/transformation_contextual.py | 1 + tests/llm_translation/test_voyage_ai.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index c4eb6f34283..4d8bcc7ff80 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -2,6 +2,7 @@ This module is used to transform the request and response for the Voyage contextualized embeddings API. This would be used for all the contextualized embeddings models in Voyage. """ + from typing import Any, Dict, List, Optional, Tuple, Union import httpx diff --git a/tests/llm_translation/test_voyage_ai.py b/tests/llm_translation/test_voyage_ai.py index a0d291bc02f..e6c73240a0e 100644 --- a/tests/llm_translation/test_voyage_ai.py +++ b/tests/llm_translation/test_voyage_ai.py @@ -300,6 +300,22 @@ class TestVoyageContextualEmbeddings: assert transformed["chunk_size"] == 32000 assert transformed["input_type"] == "document" + def test_contextual_single_string_query_no_auto_chunk(self): + """A single string with input_type='query' is not auto-chunked.""" + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + + transformed = config.transform_embedding_request( + "voyage-context-4", "Hello", {"input_type": "query"}, {} + ) + + assert transformed["inputs"] == ["Hello"] + assert transformed["input_type"] == "query" + assert "enable_auto_chunking" not in transformed + def test_contextual_nested_input_passthrough(self): """Already-nested list[list[str]] input is passed through unchanged.""" from litellm.llms.voyage.embedding.transformation_contextual import ( From bb6db24d58bb93454a77bbcb283f1f6ad4953a1d Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 18:37:25 +0200 Subject: [PATCH 015/411] fix(voyage): use modern type annotations to satisfy strict ruff budget (TID251, UP006) --- .../embedding/transformation_contextual.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 4d8bcc7ff80..8f029c27be7 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -3,7 +3,9 @@ This module is used to transform the request and response for the Voyage context This would be used for all the contextualized embeddings models in Voyage. """ -from typing import Any, Dict, List, Optional, Tuple, Union +from __future__ import annotations + +from typing import Any import httpx @@ -20,7 +22,7 @@ class VoyageError(BaseLLMException): self, status_code: int, message: str, - headers: Union[dict, httpx.Headers] = {}, + headers: dict | httpx.Headers = {}, ): self.status_code = status_code self.message = message @@ -43,12 +45,12 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): def get_complete_url( self, - api_base: Optional[str], - api_key: Optional[str], + api_base: str | None, + api_key: str | None, model: str, optional_params: dict, litellm_params: dict, - stream: Optional[bool] = None, + stream: bool | None = None, ) -> str: if api_base: if not api_base.endswith("/contextualizedembeddings"): @@ -81,11 +83,11 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): self, headers: dict, model: str, - messages: List[AllMessageValues], + messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - api_key: Optional[str] = None, - api_base: Optional[str] = None, + api_key: str | None = None, + api_base: str | None = None, ) -> dict: if api_key is None: api_key = ( @@ -105,7 +107,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): def transform_embedding_request( self, model: str, - input: Union[AllEmbeddingInputValues, List[List[str]]], + input: AllEmbeddingInputValues | list[list[str]], optional_params: dict, headers: dict, ) -> dict: @@ -120,9 +122,9 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): @classmethod def _transform_contextual_inputs( cls, - input: Union[AllEmbeddingInputValues, List[List[str]]], + input: AllEmbeddingInputValues | list[list[str]], optional_params: dict, - ) -> Tuple[Union[List[str], List[List[str]]], dict]: + ) -> tuple[list[str] | list[list[str]], dict]: """ Normalize ``input`` for Voyage's contextualized embeddings API and return ``(inputs, extra_params)`` where ``extra_params`` carries any @@ -172,7 +174,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): ``enable_auto_chunking=True`` requires ``input_type="document"``, so set it unless the caller already provided an ``input_type``. """ - params: Dict[str, Any] = { + params: dict[str, Any] = { "enable_auto_chunking": True, "chunk_size": cls.AUTO_CHUNK_SIZE, } @@ -186,7 +188,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): raw_response: httpx.Response, model_response: EmbeddingResponse, logging_obj: LiteLLMLoggingObj, - api_key: Optional[str] = None, + api_key: str | None = None, request_data: dict = {}, optional_params: dict = {}, litellm_params: dict = {}, @@ -208,9 +210,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): model_response.usage = usage return model_response - def get_error_class( - self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] - ) -> BaseLLMException: + def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return VoyageError(message=error_message, status_code=status_code, headers=headers) @staticmethod From eddb2225dc6dbc79238f4e6a7815c40cb7957a6d Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 21:29:38 +0200 Subject: [PATCH 016/411] fix(voyage): add mutable-ok pragmas and replace type: ignore with pyright: ignore (LIT001, LIT009) --- .../embedding/transformation_contextual.py | 84 ++++++++++--------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 8f029c27be7..04cb58bcfcc 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -22,11 +22,14 @@ class VoyageError(BaseLLMException): self, status_code: int, message: str, - headers: dict | httpx.Headers = {}, + headers: dict | httpx.Headers = {}, # mutable-ok: base class signature ): self.status_code = status_code self.message = message - self.request = httpx.Request(method="POST", url="https://api.voyageai.com/v1/contextualizedembeddings") + self.request = httpx.Request( + method="POST", + url="https://api.voyageai.com/v1/contextualizedembeddings", + ) self.response = httpx.Response(status_code=status_code, request=self.request) super().__init__( status_code=status_code, @@ -48,8 +51,8 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): api_base: str | None, api_key: str | None, model: str, - optional_params: dict, - litellm_params: dict, + optional_params: dict, # mutable-ok: base class signature + litellm_params: dict, # mutable-ok: base class signature stream: bool | None = None, ) -> str: if api_base: @@ -58,16 +61,16 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): return api_base return "https://api.voyageai.com/v1/contextualizedembeddings" - def get_supported_openai_params(self, model: str) -> list: + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class signature return ["encoding_format", "dimensions"] def map_openai_params( self, - non_default_params: dict, - optional_params: dict, + non_default_params: dict, # mutable-ok: base class signature + optional_params: dict, # mutable-ok: base class signature model: str, drop_params: bool, - ) -> dict: + ) -> dict: # mutable-ok: base class signature """ Map OpenAI params to Voyage params @@ -81,14 +84,14 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): def validate_environment( self, - headers: dict, + headers: dict, # mutable-ok: base class signature model: str, - messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, + messages: list[AllMessageValues], # mutable-ok: base class signature + optional_params: dict, # mutable-ok: base class signature + litellm_params: dict, # mutable-ok: base class signature api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict: # mutable-ok: base class signature if api_key is None: api_key = ( get_secret_str("VOYAGE_API_KEY") @@ -99,18 +102,15 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): "Authorization": f"Bearer {api_key}", } - # Chunk size (in tokens) used when the API auto-chunks a flat ``list[str]``. - # Matches the voyage-context-4 context window so each string stays a single - # chunk instead of being split. AUTO_CHUNK_SIZE = 32000 def transform_embedding_request( self, model: str, - input: AllEmbeddingInputValues | list[list[str]], - optional_params: dict, - headers: dict, - ) -> dict: + input: AllEmbeddingInputValues | list[list[str]], # mutable-ok: base class signature + optional_params: dict, # mutable-ok: base class signature + headers: dict, # mutable-ok: base class signature + ) -> dict: # mutable-ok: base class signature inputs, extra_params = self._transform_contextual_inputs(input, optional_params) return { "inputs": inputs, @@ -122,9 +122,9 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): @classmethod def _transform_contextual_inputs( cls, - input: AllEmbeddingInputValues | list[list[str]], - optional_params: dict, - ) -> tuple[list[str] | list[list[str]], dict]: + input: AllEmbeddingInputValues | list[list[str]], # mutable-ok: union with AllEmbeddingInputValues + optional_params: dict, # mutable-ok: matches public API + ) -> tuple[list[str] | list[list[str]], dict]: # mutable-ok: returned to caller who owns it """ Normalize ``input`` for Voyage's contextualized embeddings API and return ``(inputs, extra_params)`` where ``extra_params`` carries any @@ -149,32 +149,27 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): Reference: https://docs.voyageai.com/reference/contextualized-embeddings-api """ - # Single string -> a one-element flat list, auto-chunked. if isinstance(input, str): if optional_params.get("input_type") == "query": - return [input], {} - return [input], cls._auto_chunk_params(optional_params) + return [input], {} # mutable-ok: fresh list returned to caller + return [input], cls._auto_chunk_params(optional_params) # mutable-ok: fresh list returned to caller - # Flat list[str]. if isinstance(input, list) and all(isinstance(i, str) for i in input): if optional_params.get("input_type") == "query": - # The API accepts a flat query list as-is. - return input, {} # type: ignore[return-value] - # Otherwise let the API auto-chunk the flat list. - return input, cls._auto_chunk_params(optional_params) # type: ignore[return-value] + return input, {} # pyright: ignore[reportReturnType] # narrowed to list[str] by isinstance+all check + return input, cls._auto_chunk_params(optional_params) # pyright: ignore[reportReturnType] # narrowed to list[str] - # Already list[list[str]] (or another shape) -> pass through unchanged. - return input, {} # type: ignore[return-value] + return input, {} # pyright: ignore[reportReturnType] # list[list[str]] branch @classmethod - def _auto_chunk_params(cls, optional_params: dict) -> dict: + def _auto_chunk_params(cls, optional_params: dict) -> dict: # mutable-ok: matches public API """ Params required to send a flat ``list[str]`` to the contextualized API. ``enable_auto_chunking=True`` requires ``input_type="document"``, so set it unless the caller already provided an ``input_type``. """ - params: dict[str, Any] = { + params: dict[str, Any] = { # mutable-ok: building return value "enable_auto_chunking": True, "chunk_size": cls.AUTO_CHUNK_SIZE, } @@ -189,16 +184,18 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): model_response: EmbeddingResponse, logging_obj: LiteLLMLoggingObj, api_key: str | None = None, - request_data: dict = {}, - optional_params: dict = {}, - litellm_params: dict = {}, + request_data: dict = {}, # mutable-ok: base class signature + optional_params: dict = {}, # mutable-ok: base class signature + litellm_params: dict = {}, # mutable-ok: base class signature ) -> EmbeddingResponse: try: raw_response_json = raw_response.json() except Exception: - raise VoyageError(message=raw_response.text, status_code=raw_response.status_code) + raise VoyageError( + message=raw_response.text, + status_code=raw_response.status_code, + ) - # model_response.usage model_response.model = raw_response_json.get("model") model_response.data = raw_response_json.get("data") model_response.object = raw_response_json.get("object") @@ -210,7 +207,12 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): model_response.usage = usage return model_response - def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict | httpx.Headers, # mutable-ok: base class signature + ) -> BaseLLMException: return VoyageError(message=error_message, status_code=status_code, headers=headers) @staticmethod From af634ab4265d00f3fe730623a7f28d4ddef86d42 Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 29 Jul 2026 22:13:15 +0200 Subject: [PATCH 017/411] test(voyage): add unit tests for contextual embedding in test_litellm (CI coverage path) --- .../test_voyage_contextual_embedding.py | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py diff --git a/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py b/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py new file mode 100644 index 00000000000..865608e286f --- /dev/null +++ b/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py @@ -0,0 +1,222 @@ +import json +from unittest.mock import MagicMock + +import pytest + + +class TestVoyageContextualEmbeddings: + def test_contextual_model_detection(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + assert VoyageContextualEmbeddingConfig.is_contextualized_embeddings("voyage-context-3") + assert VoyageContextualEmbeddingConfig.is_contextualized_embeddings("voyage-context-4") + assert not VoyageContextualEmbeddingConfig.is_contextualized_embeddings("voyage-3-lite") + + def test_url_generation(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + assert ( + config.get_complete_url(None, None, "voyage-context-4", {}, {}) + == "https://api.voyageai.com/v1/contextualizedembeddings" + ) + assert ( + config.get_complete_url("https://custom.api.com", None, "voyage-context-4", {}, {}) + == "https://custom.api.com/contextualizedembeddings" + ) + assert ( + config.get_complete_url( + "https://custom.api.com/contextualizedembeddings", + None, + "voyage-context-4", + {}, + {}, + ) + == "https://custom.api.com/contextualizedembeddings" + ) + + def test_get_supported_openai_params(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + assert config.get_supported_openai_params("voyage-context-4") == [ + "encoding_format", + "dimensions", + ] + + def test_map_openai_params(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + result = config.map_openai_params( + {"encoding_format": "float", "dimensions": 512}, {}, "voyage-context-4", False + ) + assert result["encoding_format"] == "float" + assert result["output_dimension"] == 512 + + def test_validate_environment_with_api_key(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + headers = config.validate_environment( + {}, "voyage-context-4", [], {}, {}, api_key="test-key" + ) + assert headers == {"Authorization": "Bearer test-key"} + + def test_validate_environment_secret_fallback(self, monkeypatch): + import litellm.llms.voyage.embedding.transformation_contextual as module + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + def fake_get_secret(name): + return "secret-key" if name == "VOYAGE_API_KEY" else None + + monkeypatch.setattr(module, "get_secret_str", fake_get_secret) + config = VoyageContextualEmbeddingConfig() + headers = config.validate_environment( + {}, "voyage-context-4", [], {}, {}, api_key=None + ) + assert headers == {"Authorization": "Bearer secret-key"} + + def test_nested_list_passthrough(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + nested = [["Hello", "world"], ["Test"]] + transformed = config.transform_embedding_request( + "voyage-context-4", nested, {}, {} + ) + assert transformed["inputs"] == nested + assert transformed["model"] == "voyage-context-4" + assert "enable_auto_chunking" not in transformed + + def test_flat_list_str_auto_chunked(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", ["Hello", "world"], {}, {} + ) + assert transformed["inputs"] == ["Hello", "world"] + assert transformed["enable_auto_chunking"] is True + assert transformed["chunk_size"] == 32000 + assert transformed["input_type"] == "document" + + def test_flat_list_str_query_no_auto_chunk(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", ["Hello", "world"], {"input_type": "query"}, {} + ) + assert transformed["inputs"] == ["Hello", "world"] + assert transformed["input_type"] == "query" + assert "enable_auto_chunking" not in transformed + + def test_flat_list_str_document_preserves_input_type(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", ["Hello"], {"input_type": "document"}, {} + ) + assert transformed["input_type"] == "document" + assert transformed["enable_auto_chunking"] is True + + def test_single_string_auto_chunked(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", "Hello", {}, {} + ) + assert transformed["inputs"] == ["Hello"] + assert transformed["enable_auto_chunking"] is True + assert transformed["input_type"] == "document" + + def test_single_string_query_no_auto_chunk(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", "Hello", {"input_type": "query"}, {} + ) + assert transformed["inputs"] == ["Hello"] + assert transformed["input_type"] == "query" + assert "enable_auto_chunking" not in transformed + + def test_response_transformation(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + from litellm.types.utils import EmbeddingResponse + + config = VoyageContextualEmbeddingConfig() + response_payload = { + "object": "list", + "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}], + "model": "voyage-context-4", + "usage": {"total_tokens": 24}, + } + raw_response = MagicMock() + raw_response.json.return_value = response_payload + raw_response.status_code = 200 + raw_response.text = json.dumps(response_payload) + + model_response = EmbeddingResponse() + transformed = config.transform_embedding_response( + "voyage-context-4", raw_response, model_response, MagicMock() + ) + assert transformed.model == "voyage-context-4" + assert transformed.object == "list" + assert transformed.data == response_payload["data"] + assert transformed.usage.prompt_tokens == 24 + assert transformed.usage.total_tokens == 24 + + def test_error_response_and_error_class(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + VoyageError, + ) + from litellm.types.utils import EmbeddingResponse + + config = VoyageContextualEmbeddingConfig() + raw_response = MagicMock() + raw_response.json.side_effect = ValueError("not json") + raw_response.status_code = 400 + raw_response.text = "bad request" + + with pytest.raises(VoyageError) as exc_info: + config.transform_embedding_response( + "voyage-context-4", raw_response, EmbeddingResponse(), MagicMock() + ) + assert exc_info.value.status_code == 400 + assert exc_info.value.message == "bad request" + + error = config.get_error_class("rate limited", 429, {"x-test": "1"}) + assert isinstance(error, VoyageError) + assert error.status_code == 429 + assert error.message == "rate limited" From a1be4236322d7615eece57ed6cc84867e194e866 Mon Sep 17 00:00:00 2001 From: fzowl Date: Sat, 15 Aug 2026 20:29:31 +0200 Subject: [PATCH 018/411] fix(voyage): drop banned typing.Any from contextual transform to satisfy strict lint gate --- litellm/llms/voyage/embedding/transformation_contextual.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index d473450a969..0c02f00fbd5 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -3,7 +3,7 @@ This module is used to transform the request and response for the Voyage context This would be used for all the contextualized embeddings models in Voyage. """ -from typing import Any, Final +from typing import Final import httpx @@ -167,7 +167,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): ``enable_auto_chunking=True`` requires ``input_type="document"``, so set it unless the caller already provided an ``input_type``. """ - params: dict[str, Any] = { # mutable-ok: building return value + params: dict[str, object] = { # mutable-ok: building return value "enable_auto_chunking": True, "chunk_size": cls.AUTO_CHUNK_SIZE, } From 8c8cd890adcc14ed8c13d2b315e8eebc6d5c031d Mon Sep 17 00:00:00 2001 From: fzowl Date: Sat, 15 Aug 2026 20:38:40 +0200 Subject: [PATCH 019/411] fix(voyage): drop redundant isinstance list check to satisfy basedpyright budget --- litellm/llms/voyage/embedding/transformation_contextual.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 0c02f00fbd5..a9a4a01f8e3 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -152,9 +152,9 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): return [input], {} # mutable-ok: fresh list returned to caller return [input], cls._auto_chunk_params(optional_params) # mutable-ok: fresh list returned to caller - if isinstance(input, list) and all(isinstance(i, str) for i in input): + if all(isinstance(i, str) for i in input): if optional_params.get("input_type") == "query": - return input, {} # pyright: ignore[reportReturnType] # narrowed to list[str] by isinstance+all check + return input, {} # pyright: ignore[reportReturnType] # narrowed to list[str] by all(isinstance) check return input, cls._auto_chunk_params(optional_params) # pyright: ignore[reportReturnType] # narrowed to list[str] return input, {} # pyright: ignore[reportReturnType] # list[list[str]] branch From f01309c5afaf576e15170699d94185bd01b4835c Mon Sep 17 00:00:00 2001 From: ZXT-zjbiliy <3240102335@zju.edu.cn> Date: Fri, 21 Aug 2026 13:56:30 +0800 Subject: [PATCH 020/411] fix(stream_chunk_builder): guard empty choices and missing role in build_base_response build_base_response() read the assistant role via first_chunk_with_choices["choices"][0]["delta"]["role"] with no bounds or key check, causing two failures: - IndexError when no chunk carries a non-empty "choices" array, because next() fell back to the first chunk whose "choices" may be [] - KeyError when the first choice's "delta" omits "role" or is {} Both surface as "litellm.APIError: Error building chunks for logging/streaming usage calculation". async_data_generator() writes that into the response stream, so the client's answer is truncated mid-stream with no data: [DONE], and the request never reaches SpendLogs. Observed in production on Anthropic streaming. Fall back to None, guard the array length, and default the role to "assistant". The loop directly below already guards with len(chunk["choices"]) > 0. --- .../streaming_chunk_builder_utils.py | 11 +- .../test_streaming_chunk_builder_utils.py | 103 ++++++++++++++++++ 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index ee0518c4aec..59096cfaff7 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -302,8 +302,15 @@ class ChunkProcessor: model: Final = ChunkProcessor._get_model_from_chunks(chunks, first_chunk_model) system_fingerprint: Final = chunk.get("system_fingerprint", None) - first_chunk_with_choices: Final = next((c for c in chunks if c.get("choices")), chunk) - role: Final = first_chunk_with_choices["choices"][0]["delta"]["role"] + # Fall back to None rather than `chunk`: if no chunk carries a non-empty + # `choices` array, indexing [0] on the first chunk raises IndexError. + first_chunk_with_choices = next((c for c in chunks if c.get("choices")), None) + role: str = "assistant" + if first_chunk_with_choices is not None: + _choices = first_chunk_with_choices["choices"] + if len(_choices) > 0: + # `delta` may be absent or omit `role` (e.g. content-only deltas). + role = _choices[0].get("delta", {}).get("role") or "assistant" finish_reason = "stop" for chunk in chunks: if "choices" in chunk and len(chunk["choices"]) > 0: diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 0f21cce476b..aec189da653 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1342,3 +1342,106 @@ def test_calculate_usage_fills_unknown_split_from_reasoning_estimate( assert usage.completion_tokens == 100 assert usage.completion_tokens_details.reasoning_tokens == expected_reasoning_tokens assert usage.completion_tokens_details.text_tokens == expected_text_tokens + + +def _empty_choices_chunk(**extra): + chunk = { + "id": "chatcmpl-empty-choices", + "object": "chat.completion.chunk", + "created": 1, + "model": "claude-opus-4-8", + "choices": [], + } + chunk.update(extra) + return chunk + + +@pytest.mark.parametrize( + "chunks", + [ + pytest.param( + [_empty_choices_chunk(), _empty_choices_chunk()], + id="all_chunks_have_empty_choices", + ), + pytest.param( + [ + _empty_choices_chunk(usage={"prompt_tokens": 10}), + _empty_choices_chunk(usage={"completion_tokens": 0}), + ], + id="usage_only_chunks", + ), + ], +) +def test_build_base_response_handles_empty_choices(chunks): + """Empty `choices` arrays must not raise IndexError. + + `next((c for c in chunks if c.get("choices")), chunk)` used to fall back to the + first chunk, whose `choices` may be `[]`, so `["choices"][0]` went out of range. + The resulting error is surfaced to the client mid-stream and the request never + reaches SpendLogs. + """ + processor = ChunkProcessor(chunks=list(chunks)) + + response = processor.build_base_response(list(chunks)) + + assert response.choices[0].message.role == "assistant" + + +@pytest.mark.parametrize( + "delta", + [ + pytest.param({"content": "Hello"}, id="delta_without_role"), + pytest.param({}, id="delta_empty_dict"), + ], +) +def test_build_base_response_handles_delta_without_role(delta): + """A `delta` that omits `role` must not raise KeyError.""" + chunks = [ + { + "id": "chatcmpl-no-role", + "object": "chat.completion.chunk", + "created": 1, + "model": "claude-opus-4-8", + "choices": [{"index": 0, "delta": delta, "finish_reason": None}], + } + ] + processor = ChunkProcessor(chunks=list(chunks)) + + response = processor.build_base_response(list(chunks)) + + assert response.choices[0].message.role == "assistant" + + +def test_build_base_response_still_reads_role_and_finish_reason(): + """Regression guard: well-formed chunks keep their role and finish_reason.""" + chunks = [ + _empty_choices_chunk(), + { + "id": "chatcmpl-normal", + "object": "chat.completion.chunk", + "created": 1, + "model": "claude-opus-4-8", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hi"}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-normal", + "object": "chat.completion.chunk", + "created": 2, + "model": "claude-opus-4-8", + "choices": [ + {"index": 0, "delta": {"content": "!"}, "finish_reason": "stop"} + ], + }, + ] + processor = ChunkProcessor(chunks=list(chunks)) + + response = processor.build_base_response(list(chunks)) + + assert response.choices[0].message.role == "assistant" + assert response.choices[0].finish_reason == "stop" From 88b3cfd789b5726af3acb7387dc7acb54dbacd85 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:55:43 +0000 Subject: [PATCH 021/411] fix(mcp): bind per-user OAuth credentials to the authenticated LiteLLM caller Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/discoverable_endpoints.py | 20 +- .../mcp_server/mcp_server_manager.py | 2 + .../mcp_server/oauth_identity_binding.py | 249 ++++++++++++++++++ .../mcp_management_endpoints.py | 22 ++ .../types/mcp_server/mcp_server_manager.py | 21 ++ .../mcp_server/test_oauth_identity_binding.py | 240 +++++++++++++++++ .../test_mcp_management_endpoints.py | 61 +++++ 7 files changed, 614 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 93b85edd88d..370dee77083 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -54,6 +54,9 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( relative_request_url, revoke_refresh_token, ) +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + enforce_oauth_identity_binding, +) from litellm.proxy._experimental.mcp_server.oauth_utils import ( TOKEN_NO_CACHE_HEADERS, build_upstream_oauth2_token_request, @@ -1133,11 +1136,26 @@ async def exchange_token_with_server( server_id=resolved_server.server_id, ) + # Bind the exchanged token to the LiteLLM caller BEFORE it is returned, stored, or cached, so a + # token minted for a different upstream principal never becomes usable under the caller's user_id. + resolved_user_id: Final = ( + await _extract_user_id_from_request(request) + if resolved_server.needs_user_oauth_token or resolved_server.oauth_identity_binding is not None + else None + ) + if isinstance(token_response, dict): + await enforce_oauth_identity_binding( + server=resolved_server, + token_response=token_response, + litellm_user_id=resolved_user_id, + grant_type=grant_type, + ) + # Store server-side when the server is configured for per-user OAuth and # the calling client has provided a valid LiteLLM identity. # Errors are non-fatal: the token is still returned to the client. if resolved_server.needs_user_oauth_token: - user_id: Final = await _extract_user_id_from_request(request) + user_id: Final = resolved_user_id if user_id: try: await _store_per_user_token_server_side( diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 2330120adad..1d61aba8bc5 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -2174,6 +2174,8 @@ class MCPServerManager: allow_elicitation=bool(server_config.get("allow_elicitation", False)), timeout=server_config.get("timeout", None), max_concurrent_requests=server_config.get("max_concurrent_requests", None), + token_validation=server_config.get("token_validation", None), + oauth_identity_binding=server_config.get("oauth_identity_binding", None), ) self._assign_unique_short_prefix(new_server) _warn_internal_delegate_pkce_if_applicable(new_server, source="config") diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py new file mode 100644 index 00000000000..f132f59ae99 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -0,0 +1,249 @@ +"""Per-user OAuth identity binding: verify the upstream OIDC principal matches the LiteLLM caller. + +Closes the confused-deputy gap where a browser authenticated upstream as one principal produces a +token that the relay stores under a different, LiteLLM-authenticated principal: before the token +endpoint returns, stores, or caches an exchanged token for an identity-bound server, the id_token +is validated (signature via the pinned issuer's JWKS, issuer, audience, expiry) and its principal +claim is compared to the caller's trusted LiteLLM identity. Mismatches fail closed in enforce mode +and are logged in audit mode. +""" + +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from typing import Final, Literal + +import jwt +from fastapi import HTTPException + +from litellm._logging import verbose_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + +_ALLOWED_ID_TOKEN_ALGORITHMS: Final = ( + "RS256", + "RS384", + "RS512", + "ES256", + "ES384", + "ES512", + "PS256", + "PS384", + "PS512", +) +_JWKS_CACHE_TTL_SECONDS: Final = 3600 +_jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) + +JwksFetcher = Callable[[MCPOAuthIdentityBinding], Awaitable[list[Mapping[str, object]]]] +CallerPrincipalLoader = Callable[[str, MCPOAuthIdentityBinding], Awaitable[str | None]] + +_RejectionCode = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] + + +@dataclass(frozen=True, slots=True) +class _BindingRejection: + code: _RejectionCode + description: str + + +async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: + jwks_url: Final[str] = binding.jwks_url or await _discover_jwks_url(binding.issuer) + cached: Final = await _jwks_cache.async_get_cache(jwks_url) + if isinstance(cached, list): + return cached + client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) + response: Final = await client.get(jwks_url) + response.raise_for_status() + document: Final = response.json() + keys: Final = document.get("keys") if isinstance(document, dict) else None + if not isinstance(keys, list): + raise TypeError(f"JWKS document at {jwks_url} has no 'keys' array") + await _jwks_cache.async_set_cache(jwks_url, keys, ttl=_JWKS_CACHE_TTL_SECONDS) + return keys + + +async def _discover_jwks_url(issuer: str) -> str: + discovery_url: Final = f"{issuer.rstrip('/')}/.well-known/openid-configuration" + client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) + response: Final = await client.get(discovery_url) + response.raise_for_status() + metadata: Final = response.json() + jwks_uri: Final = metadata.get("jwks_uri") if isinstance(metadata, dict) else None + if not isinstance(jwks_uri, str) or not jwks_uri: + raise ValueError(f"OIDC discovery at {discovery_url} returned no jwks_uri") + return jwks_uri + + +def _select_signing_key(id_token: str, keys: list[Mapping[str, object]]) -> "jwt.PyJWK | _BindingRejection": + header: Final = jwt.get_unverified_header(id_token) + kid: Final = header.get("kid") + for key in keys: + if kid is None or key.get("kid") == kid: + return jwt.PyJWK(dict(key)) + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"id_token signing key (kid={kid!r}) not found in the issuer's JWKS", + ) + + +def _decode_id_token( + id_token: str, + binding: MCPOAuthIdentityBinding, + signing_key: "jwt.PyJWK", +) -> "Mapping[str, object] | _BindingRejection": + try: + return jwt.decode( + id_token, + signing_key.key, + algorithms=list(_ALLOWED_ID_TOKEN_ALGORITHMS), + issuer=binding.issuer, + audience=binding.audiences if binding.audiences else None, + options={ + "require": ["iss", "exp"], + "verify_aud": bool(binding.audiences), + }, + ) + except jwt.InvalidTokenError as exc: + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"id_token validation failed: {exc}", + ) + + +def _upstream_principal( + claims: Mapping[str, object], + binding: MCPOAuthIdentityBinding, +) -> "str | _BindingRejection": + principal: Final = claims.get(binding.principal_claim) + if not isinstance(principal, str) or not principal: + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"id_token has no usable '{binding.principal_claim}' claim", + ) + if binding.principal_claim == "email" and binding.require_email_verified and claims.get("email_verified") is not True: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="id_token email is not verified (email_verified is not true)", + ) + return principal + + +async def _load_caller_principal(litellm_user_id: str, binding: MCPOAuthIdentityBinding) -> str | None: + if binding.caller_field == "user_id": + return litellm_user_id + from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # inline import avoids a module-load circular import + load_active_user_by_id, + ) + + loaded: Final = await load_active_user_by_id(litellm_user_id) + if isinstance(loaded, str): + return None + return loaded.user_email + + +def _principals_match(upstream: str, caller: str, binding: MCPOAuthIdentityBinding) -> bool: + if binding.principal_claim == "email" or binding.caller_field == "user_email": + return upstream.strip().casefold() == caller.strip().casefold() + return upstream == caller + + +async def _evaluate_binding( + binding: MCPOAuthIdentityBinding, + token_response: Mapping[str, object], + litellm_user_id: str | None, + grant_type: str, + jwks_fetcher: JwksFetcher, + caller_principal_loader: CallerPrincipalLoader, +) -> _BindingRejection | None: + id_token: Final = token_response.get("id_token") + if not isinstance(id_token, str) or not id_token: + if grant_type == "refresh_token": + return None + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the upstream token response carries no id_token to bind the credential to a principal", + ) + if not litellm_user_id: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the request carries no resolvable LiteLLM user identity to bind the credential to", + ) + try: + keys: Final = await jwks_fetcher(binding) + except Exception as exc: # noqa: BLE001 # a JWKS fetch failure must fail closed, not surface as a 500 + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"could not fetch the issuer's JWKS: {exc}", + ) + signing_key: Final = _select_signing_key(id_token, keys) + if isinstance(signing_key, _BindingRejection): + return signing_key + claims: Final = _decode_id_token(id_token, binding, signing_key) + if isinstance(claims, _BindingRejection): + return claims + upstream: Final = _upstream_principal(claims, binding) + if isinstance(upstream, _BindingRejection): + return upstream + caller: Final = await caller_principal_loader(litellm_user_id, binding) + if not caller: + return _BindingRejection( + code="oauth_identity_binding_failed", + description=f"the LiteLLM user has no '{binding.caller_field}' to compare the upstream principal against", + ) + if not _principals_match(upstream, caller, binding): + return _BindingRejection( + code="oauth_principal_mismatch", + description="The browser account does not match the selected credential owner.", + ) + return None + + +async def enforce_oauth_identity_binding( + server: MCPServer, + token_response: Mapping[str, object], + litellm_user_id: str | None, + grant_type: str, + jwks_fetcher: JwksFetcher = _fetch_issuer_jwks, + caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, +) -> None: + """Validate the exchanged token's upstream principal against the LiteLLM caller. + + No-op when the server has no binding or it is disabled. In enforce mode a failure raises 403 + before the caller returns, stores, or caches the token; in audit mode failures are logged only. + A refresh_token grant without an id_token is allowed in both modes: the stored credential keeps + the binding established at the original authorization_code exchange. + """ + binding: Final = server.oauth_identity_binding + if binding is None or binding.mode == "disabled": + return + rejection: Final = await _evaluate_binding( + binding=binding, + token_response=token_response, + litellm_user_id=litellm_user_id, + grant_type=grant_type, + jwks_fetcher=jwks_fetcher, + caller_principal_loader=caller_principal_loader, + ) + if rejection is None: + return + if binding.mode == "audit": + verbose_logger.warning( + "oauth_identity_binding audit: server=%s user=%s grant=%s rejected=%s (%s)", + server.server_id, + litellm_user_id, + grant_type, + rejection.code, + rejection.description, + ) + return + raise HTTPException( + status_code=403, + detail={ + "error": rejection.code, + "error_description": rejection.description, + "server_id": server.server_id, + "credential_owner": "caller", + "credential_stored": False, + }, + ) diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 556a30d0b29..5e68e2440da 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -2135,6 +2135,28 @@ if MCP_AVAILABLE: """Persist the OAuth2 access token obtained by the calling user.""" prisma_client: Final = get_prisma_client_or_throw("Database not connected. Connect a database to your proxy") await _authorize_and_fetch_mcp_server(prisma_client, user_api_key_dict, server_id) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + global_mcp_server_manager as _manager, + ) + + # This endpoint accepts an opaque token with no upstream identity validation, so it must be + # closed for identity-bound servers or it becomes a bypass of the token-relay binding check. + registry_server: Final = _manager.get_mcp_server_by_id(server_id) + binding: Final = registry_server.oauth_identity_binding if registry_server else None + if binding is not None and binding.mode == "enforce": + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": "oauth_identity_binding_enforced", + "error_description": ( + "Direct credential storage is disabled for this server: its OAuth identity " + "binding is enforced and this endpoint cannot validate the token's principal. " + "Complete the OAuth flow through the gateway instead." + ), + "server_id": server_id, + "credential_stored": False, + }, + ) user_id: Final = user_api_key_dict.user_id or "" if not user_id: raise HTTPException( diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 9bf3acc601c..cc3f6c8e8d8 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -38,6 +38,26 @@ class MCPOAuthMetadata(BaseModel): usable in memory but must never be persisted as configuration.""" +class MCPOAuthIdentityBinding(BaseModel): + """Per-server policy binding stored per-user OAuth credentials to the authenticated LiteLLM caller. + + When enabled for an interactive oauth2 server, the token relay validates the upstream OIDC + ``id_token`` (signature via the pinned issuer's JWKS, issuer, audience, expiry) and compares its + principal claim to the LiteLLM caller's trusted identity before the token is returned, stored, + or cached. ``audit`` logs mismatches without changing behavior; ``enforce`` fails closed with + 403 ``oauth_principal_mismatch`` and disables the direct ``oauth-user-credential`` POST, which + would otherwise bypass validation with an arbitrary opaque token. + """ + + mode: Literal["disabled", "audit", "enforce"] = "disabled" + issuer: str + jwks_url: str | None = None + audiences: list[str] = [] + principal_claim: str = "email" + caller_field: Literal["user_email", "user_id"] = "user_email" + require_email_verified: bool = True + + class MCPServer(BaseModel): server_id: str name: str @@ -172,6 +192,7 @@ class MCPServer(BaseModel): # response (supports dot-notation for nested fields, e.g. "team.enterprise_id"). # Tokens that fail validation are rejected before storage. token_validation: dict[str, Any] | None = None + oauth_identity_binding: MCPOAuthIdentityBinding | None = None # Optional TTL override (seconds) for the Redis per-user token cache, capped # at the token's expires_in minus the expiry buffer so a cached entry never # outlives the token. Defaults to the token's expires_in minus the expiry diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py new file mode 100644 index 00000000000..69eaabeda4a --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -0,0 +1,240 @@ +import time +from collections.abc import Mapping +from typing import Final + +import jwt +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from fastapi import HTTPException + +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + enforce_oauth_identity_binding, +) +from litellm.types.mcp import MCPTransport +from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + +ISSUER: Final = "https://idp.example.com" +AUDIENCE: Final = "litellm-client" +KID: Final = "test-key" + +_PRIVATE_KEY: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) +_PRIVATE_PEM: Final = _PRIVATE_KEY.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), +) +_PUBLIC_JWK: Final = { + **jwt.algorithms.RSAAlgorithm.to_jwk(_PRIVATE_KEY.public_key(), as_dict=True), + "kid": KID, + "alg": "RS256", + "use": "sig", +} + + +def _sign_id_token(claims: Mapping[str, object]) -> str: + payload: Final = { + "iss": ISSUER, + "aud": AUDIENCE, + "exp": int(time.time()) + 300, + "iat": int(time.time()), + **claims, + } + return jwt.encode(payload, _PRIVATE_PEM, algorithm="RS256", headers={"kid": KID}) + + +async def _jwks_fetcher(_binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: + return [_PUBLIC_JWK] + + +def _caller_loader(email: str | None): + async def load(_user_id: str, _binding: MCPOAuthIdentityBinding) -> str | None: + return email + + return load + + +def _server(mode: str = "enforce", **binding_overrides: object) -> MCPServer: + return MCPServer( + server_id="srv-1", + name="srv-1", + url="https://mcp.example.com", + transport=MCPTransport.http, + oauth_identity_binding=MCPOAuthIdentityBinding( + mode=mode, + issuer=ISSUER, + audiences=[AUDIENCE], + **binding_overrides, + ), + ) + + +@pytest.mark.asyncio +async def test_matching_principal_passes(): + token: Final = _sign_id_token({"email": "Alice@Example.com", "email_verified": True}) + result: Final = await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_mismatched_principal_rejected(): + token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_principal_mismatch" + assert exc_info.value.detail["credential_stored"] is False + + +@pytest.mark.asyncio +async def test_missing_id_token_rejected_on_authorization_code(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_allowed(): + result: Final = await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_refresh_with_mismatched_id_token_rejected(): + token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) + with pytest.raises(HTTPException): + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="refresh_token", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + + +@pytest.mark.asyncio +async def test_audit_mode_logs_but_does_not_reject(): + token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) + result: Final = await enforce_oauth_identity_binding( + server=_server(mode="audit"), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_unverified_email_rejected(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": False}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_wrong_issuer_rejected(): + payload: Final = { + "iss": "https://evil.example.com", + "aud": AUDIENCE, + "exp": int(time.time()) + 300, + "email": "alice@example.com", + "email_verified": True, + } + token: Final = jwt.encode(payload, _PRIVATE_PEM, algorithm="RS256", headers={"kid": KID}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_no_litellm_identity_rejected(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id=None, + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_disabled_binding_is_noop(): + result: Final = await enforce_oauth_identity_binding( + server=_server(mode="disabled"), + token_response={"access_token": "at"}, + litellm_user_id=None, + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader(None), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_no_binding_is_noop(): + server: Final = MCPServer( + server_id="srv-2", + name="srv-2", + url="https://mcp.example.com", + transport=MCPTransport.http, + ) + result: Final = await enforce_oauth_identity_binding( + server=server, + token_response={"access_token": "at"}, + litellm_user_id=None, + grant_type="authorization_code", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader(None), + ) + assert result is None diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index ceb44de5576..1cb36918f11 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -4638,6 +4638,67 @@ async def test_store_mcp_oauth_user_credential_returns_status(): assert result.expires_at == "2099-01-01T00:00:00+00:00" +@pytest.mark.asyncio +async def test_store_mcp_oauth_user_credential_blocked_when_identity_binding_enforced(): + """The direct opaque-token POST must be closed for enforce-mode identity-bound servers, + otherwise it bypasses the token-relay principal check.""" + from litellm.proxy._types import MCPOAuthUserCredentialRequest + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + if not mgmt_endpoints.MCP_AVAILABLE: + pytest.skip("MCP module not installed") + + from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + store_mcp_oauth_user_credential, + ) + + server_id = "srv-binding-1" + bound_server = MCPServer( + server_id=server_id, + name=server_id, + url="https://mcp.example.com", + transport=MCPTransport.http, + oauth_identity_binding=MCPOAuthIdentityBinding(mode="enforce", issuer="https://idp.example.com"), + ) + store_mock = AsyncMock(return_value=None) + + with ( + patch( # test-quality-ok: mirrors the existing store-credential tests in this file + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=_make_prisma_client(), + ), + patch( # test-quality-ok: mirrors the existing store-credential tests in this file + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + new=AsyncMock(return_value=generate_mock_mcp_server_db_record(server_id=server_id)), + ), + patch( # test-quality-ok: mirrors the existing store-credential tests in this file + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=True, + ), + patch.object( # test-quality-ok: registry is a module-level singleton; injecting it would change the endpoint signature + manager_module.global_mcp_server_manager, + "get_mcp_server_by_id", + return_value=bound_server, + ), + patch( # test-quality-ok: asserting the DB write is never reached is the point of the test + "litellm.proxy.management_endpoints.mcp_management_endpoints.store_user_oauth_credential", + new=store_mock, + ), + ): + with pytest.raises(HTTPException) as exc_info: + await store_mcp_oauth_user_credential( + server_id=server_id, + payload=MCPOAuthUserCredentialRequest(access_token="opaque-tok", expires_in=3600), + user_api_key_dict=_make_user_auth("user-123"), + ) + + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_enforced" + store_mock.assert_not_called() + + @pytest.mark.asyncio async def test_delete_mcp_oauth_user_credential_only_deletes_oauth(): """delete_mcp_oauth_user_credential only deletes OAuth2 credentials, not BYOK.""" From 997a813ab99d6bd4a6b8d52fd9b42e5945c409d4 Mon Sep 17 00:00:00 2001 From: Wolfram Ravenwolf Date: Tue, 1 Sep 2026 23:15:18 +0200 Subject: [PATCH 022/411] fix(wandb): preserve reasoning_effort in chat completions --- litellm/llms/wandb/chat/transformation.py | 3 + .../wandb/test_wandb_chat_transformation.py | 55 ++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/litellm/llms/wandb/chat/transformation.py b/litellm/llms/wandb/chat/transformation.py index a477c00f643..cb9ac7b4c9b 100644 --- a/litellm/llms/wandb/chat/transformation.py +++ b/litellm/llms/wandb/chat/transformation.py @@ -10,6 +10,9 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig class WandbConfig(OpenAIGPTConfig): + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: inherited contract + return super().get_supported_openai_params(model) + ["reasoning_effort"] # mutable-ok: inherited contract + def map_openai_params( self, non_default_params: dict, diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index a5d1eccebe0..4fd66448009 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -5,7 +5,7 @@ These tests validate the WandbInferenceConfig class which extends OpenAIGPTConfi Nebius AI Studio is an OpenAI-compatible provider with minor customizations. """ - +import json import pytest @@ -17,6 +17,20 @@ from litellm.llms.wandb.chat.transformation import WandbConfig class TestWandbConfig: """Test class for WandB Inference functionality""" + def test_map_openai_params_preserves_reasoning_effort(self): + supported_params = litellm.get_supported_openai_params(model="wandb/openai/gpt-oss-20b") + assert supported_params is not None + assert "reasoning_effort" in supported_params + + result = WandbConfig().map_openai_params( + non_default_params={"reasoning_effort": "medium"}, + optional_params={}, + model="openai/gpt-oss-20b", + drop_params=True, + ) + + assert result == {"reasoning_effort": "medium"} + def test_default_api_base(self): """Test that default API base is used when none is provided""" config = WandbConfig() @@ -139,3 +153,42 @@ class TestWandbConfig: # Check for specific content in the response assert "```python" in content assert "Hey from LiteLLM" in content + + @pytest.mark.respx() + def test_wandb_completion_preserves_reasoning_effort_with_drop_params(self, respx_mock, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + api_base = "https://api.inference.wandb.ai/v1" + request_mock = respx_mock.post(f"{api_base}/chat/completions").respond( + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-oss-20b", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Done"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + status_code=200, + ) + + completion( + model="wandb/openai/gpt-oss-20b", + messages=[{"role": "user", "content": "Hello"}], + api_key="fake-wandb-key", + api_base=api_base, + reasoning_effort="medium", + drop_params=True, + ) + + request_body = json.loads(request_mock.calls[0].request.content) + assert request_body["reasoning_effort"] == "medium" From 269f2336ec9776c9455044268457c23a3aa228c6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:19:14 -0700 Subject: [PATCH 023/411] fix(dashscope): remap chat-shaped api_base to the live rerank route DashScope serves rerank at /compatible-api/v1/reranks, but the chat default and DASHSCOPE_API_BASE both hand rerank a /compatible-mode/v1 base, which turned into the dead /compatible-mode/v1/reranks route. Chat-shaped .aliyuncs.com bases now remap to the same host's rerank route, preserving the mainland/intl region that API keys are scoped to. Explicit rerank api_base values and the *_API_BASE_RERANK env vars still win. Applies to dashscope, qwencloud, and qwen_ai_platform. --- litellm/llms/dashscope/common_utils.py | 24 ++++++- litellm/llms/dashscope/qwen_ai_platform.py | 5 +- litellm/llms/dashscope/qwencloud.py | 5 +- .../llms/dashscope/rerank/transformation.py | 14 ++-- .../test_dashscope_rerank_transformation.py | 32 +++++++-- .../llms/dashscope/test_qwen_brand_aliases.py | 5 ++ tests/test_litellm/rerank_api/test_main.py | 66 +++++++++++++++++++ 7 files changed, 138 insertions(+), 13 deletions(-) diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index b7c97893a15..9ed9c276e43 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -2,7 +2,8 @@ Common utilities for the DashScope LLM provider. """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final +from urllib.parse import urlparse import httpx @@ -16,6 +17,27 @@ if TYPE_CHECKING: ) from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +DASHSCOPE_CHAT_COMPATIBLE_PATH: Final = "/compatible-mode/v1" +DASHSCOPE_RERANK_PATH: Final = "/compatible-api/v1/reranks" + + +def _rerank_base_for_chat_shaped_base(api_base: str | None) -> str | None: + if api_base is None: + return None + parsed: Final = urlparse(api_base) + host: Final = parsed.hostname or "" + on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com") + if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: + return None + return f"{parsed.scheme}://{parsed.netloc}{DASHSCOPE_RERANK_PATH}" + + +def resolve_dashscope_family_rerank_api_base(api_base: str | None, env_var: str, default_rerank_base: str) -> str: + remapped: Final = _rerank_base_for_chat_shaped_base(api_base) + if api_base is not None and remapped is None: + return api_base + return get_secret_str(env_var) or remapped or default_rerank_base + def get_dashscope_family_embedding_config(custom_llm_provider: str) -> "BaseEmbeddingConfig": if custom_llm_provider == "qwencloud": diff --git a/litellm/llms/dashscope/qwen_ai_platform.py b/litellm/llms/dashscope/qwen_ai_platform.py index 9a44eaf574a..6998a57e2b7 100644 --- a/litellm/llms/dashscope/qwen_ai_platform.py +++ b/litellm/llms/dashscope/qwen_ai_platform.py @@ -3,6 +3,7 @@ from typing import Final from litellm.secret_managers.main import get_secret_str from .chat.transformation import DashScopeChatConfig +from .common_utils import resolve_dashscope_family_rerank_api_base from .embed.transformation import DashScopeEmbeddingConfig from .image_generation.transformation import DashScopeImageGenerationConfig from .rerank.transformation import DashScopeRerankConfig @@ -51,7 +52,9 @@ class QwenAIPlatformRerankConfig(DashScopeRerankConfig): return _require_qwen_ai_platform_api_key(api_key) def _resolve_rerank_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE_RERANK") or QWEN_AI_PLATFORM_RERANK_API_BASE + return resolve_dashscope_family_rerank_api_base( + api_base, "QWEN_AI_PLATFORM_API_BASE_RERANK", QWEN_AI_PLATFORM_RERANK_API_BASE + ) class QwenAIPlatformImageGenerationConfig(DashScopeImageGenerationConfig): diff --git a/litellm/llms/dashscope/qwencloud.py b/litellm/llms/dashscope/qwencloud.py index d8d53e340ef..ac21a48dd3e 100644 --- a/litellm/llms/dashscope/qwencloud.py +++ b/litellm/llms/dashscope/qwencloud.py @@ -3,6 +3,7 @@ from typing import Final from litellm.secret_managers.main import get_secret_str from .chat.transformation import DashScopeChatConfig +from .common_utils import resolve_dashscope_family_rerank_api_base from .embed.transformation import DashScopeEmbeddingConfig from .image_generation.transformation import DashScopeImageGenerationConfig from .rerank.transformation import DashScopeRerankConfig @@ -51,7 +52,9 @@ class QwenCloudRerankConfig(DashScopeRerankConfig): return _require_qwencloud_api_key(api_key) def _resolve_rerank_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWENCLOUD_API_BASE_RERANK") or QWENCLOUD_RERANK_API_BASE + return resolve_dashscope_family_rerank_api_base( + api_base, "QWENCLOUD_API_BASE_RERANK", QWENCLOUD_RERANK_API_BASE + ) class QwenCloudImageGenerationConfig(DashScopeImageGenerationConfig): diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index 3dd3996b2ee..abfe55b8d7a 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -12,8 +12,12 @@ Endpoint - https://dashscope.aliyuncs.com/compatible-api/v1/reranks Note: chat/embed live under `/compatible-mode/v1/`, but DashScope's rerank -route is exposed under `/compatible-api/v1/reranks` per the docs. Override -with `DASHSCOPE_API_BASE_RERANK` to point at a different host or path. +route is exposed under `/compatible-api/v1/reranks` per the docs. A chat-shaped +`.aliyuncs.com/compatible-mode/v1` base reaching this config (the chat default +from `get_llm_provider`, or a `DASHSCOPE_API_BASE` env var) is redirected to +the same host's rerank route, since `/compatible-mode/v1/reranks` is a dead +route on every DashScope host. Override with `DASHSCOPE_API_BASE_RERANK` to +point at a different host or path. Empirically, qwen3-rerank accepts `return_documents=true` and echoes `results[].document.text` back, even though the public docs list the flag @@ -40,7 +44,7 @@ from litellm.types.rerank import ( RerankTokens, ) -from ..common_utils import DashScopeError +from ..common_utils import DashScopeError, resolve_dashscope_family_rerank_api_base DEFAULT_RERANK_URL: Final = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks" @@ -67,9 +71,7 @@ class DashScopeRerankConfig(BaseRerankConfig): return resolved_api_key def _resolve_rerank_api_base(self, api_base: str | None) -> str: - if api_base is not None: - return api_base - return get_secret_str("DASHSCOPE_API_BASE_RERANK") or DEFAULT_RERANK_URL + return resolve_dashscope_family_rerank_api_base(api_base, "DASHSCOPE_API_BASE_RERANK", DEFAULT_RERANK_URL) def get_complete_url( self, diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py index 936de812bc6..4466e5b8767 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py @@ -25,19 +25,43 @@ class TestDashScopeRerankURL: url = self.config.get_complete_url(api_base=None, model="qwen3-rerank") assert url == DEFAULT_RERANK_URL - def test_explicit_v1_base_appends_reranks(self): + def test_chat_shaped_base_remaps_to_rerank_route(self): url = self.config.get_complete_url( api_base="https://dashscope.aliyuncs.com/compatible-mode/v1", model="qwen3-rerank", ) - assert url == "https://dashscope.aliyuncs.com/compatible-mode/v1/reranks" + assert url == "https://dashscope.aliyuncs.com/compatible-api/v1/reranks" - def test_intl_v1_base_appends_reranks(self): + def test_intl_chat_shaped_base_remaps_to_intl_rerank_route(self): url = self.config.get_complete_url( api_base="https://dashscope-intl.aliyuncs.com/compatible-mode/v1", model="qwen3-rerank", ) - assert url == "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/reranks" + assert url == "https://dashscope-intl.aliyuncs.com/compatible-api/v1/reranks" + + def test_chat_shaped_base_with_trailing_slash_remaps(self): + url = self.config.get_complete_url( + api_base="https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", + model="qwen3-rerank", + ) + assert url == "https://dashscope-intl.aliyuncs.com/compatible-api/v1/reranks" + + def test_rerank_env_var_wins_over_chat_shaped_base(self, monkeypatch): + monkeypatch.setenv( + "DASHSCOPE_API_BASE_RERANK", "https://rerank.example.com/v1/reranks" + ) + url = self.config.get_complete_url( + api_base="https://dashscope-intl.aliyuncs.com/compatible-mode/v1", + model="qwen3-rerank", + ) + assert url == "https://rerank.example.com/v1/reranks" + + def test_non_aliyun_chat_path_base_not_remapped(self): + url = self.config.get_complete_url( + api_base="https://gateway.example.com/compatible-mode/v1", + model="qwen3-rerank", + ) + assert url == "https://gateway.example.com/compatible-mode/v1/reranks" def test_already_complete_url_passthrough(self): full = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks" diff --git a/tests/test_litellm/llms/dashscope/test_qwen_brand_aliases.py b/tests/test_litellm/llms/dashscope/test_qwen_brand_aliases.py index 064d9d58f0c..7862297bcd5 100644 --- a/tests/test_litellm/llms/dashscope/test_qwen_brand_aliases.py +++ b/tests/test_litellm/llms/dashscope/test_qwen_brand_aliases.py @@ -209,6 +209,11 @@ class TestQwenBrandDefaultUrls: url = brand["rerank_config"]().get_complete_url(api_base=None, model="gte-rerank-v2") assert url == "https://rerank.example.com/v1/reranks" + @pytest.mark.parametrize("brand", BRAND_CASES) + def test_rerank_remaps_chat_shaped_default_base(self, brand): + url = brand["rerank_config"]().get_complete_url(api_base=brand["default_base"], model="gte-rerank-v2") + assert url == brand["default_rerank_base"] + @pytest.mark.parametrize("brand", BRAND_CASES) def test_image_generation_complete_url(self, brand): url = brand["image_config"]().get_complete_url( diff --git a/tests/test_litellm/rerank_api/test_main.py b/tests/test_litellm/rerank_api/test_main.py index 587be59c550..1051198c904 100644 --- a/tests/test_litellm/rerank_api/test_main.py +++ b/tests/test_litellm/rerank_api/test_main.py @@ -111,6 +111,72 @@ def test_together_rerank_honors_api_base(respx_mock: respx.MockRouter): assert mock_route.calls[0].request.headers["authorization"] == "Bearer fake-together-key" +DASHSCOPE_RERANK_BODY = { + "object": "list", + "results": [{"index": 0, "relevance_score": 0.95}], + "model": "qwen3-rerank", + "id": "rerank-mock-id", + "usage": {"total_tokens": 10}, +} + + +def test_dashscope_rerank_defaults_to_live_rerank_route(respx_mock: respx.MockRouter, monkeypatch): + """Regression for the dead default endpoint: get_llm_provider always returns the + chat base for dashscope, which used to hijack rerank onto the dead + /compatible-mode/v1/reranks route.""" + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) + + mock_route = respx_mock.post("https://dashscope.aliyuncs.com/compatible-api/v1/reranks") + mock_route.return_value = httpx.Response(200, json=DASHSCOPE_RERANK_BODY) + + response = litellm.rerank( + model="dashscope/qwen3-rerank", + query=MARKER_QUERY, + documents=[MARKER_DOC], + api_key="fake-dashscope-key", + ) + + assert mock_route.called + assert response.results[0]["relevance_score"] == 0.95 + + +def test_dashscope_rerank_chat_env_base_keeps_host_and_rerank_route(respx_mock: respx.MockRouter, monkeypatch): + """Regression: a chat-style DASHSCOPE_API_BASE must not hijack rerank onto the + chat path, while its host (the region) is preserved.""" + monkeypatch.setenv("DASHSCOPE_API_BASE", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1") + monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) + + mock_route = respx_mock.post("https://dashscope-intl.aliyuncs.com/compatible-api/v1/reranks") + mock_route.return_value = httpx.Response(200, json=DASHSCOPE_RERANK_BODY) + + litellm.rerank( + model="dashscope/qwen3-rerank", + query=MARKER_QUERY, + documents=[MARKER_DOC], + api_key="fake-dashscope-key", + ) + + assert mock_route.called + + +def test_dashscope_rerank_explicit_api_base_wins(respx_mock: respx.MockRouter, monkeypatch): + monkeypatch.setenv("DASHSCOPE_API_BASE", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1") + + mock_route = respx_mock.post("https://custom-rerank.example/v1/reranks") + mock_route.return_value = httpx.Response(200, json=DASHSCOPE_RERANK_BODY) + + litellm.rerank( + model="dashscope/qwen3-rerank", + query=MARKER_QUERY, + documents=[MARKER_DOC], + api_key="fake-dashscope-key", + api_base="https://custom-rerank.example/v1", + ) + + assert mock_route.called + + @pytest.mark.asyncio async def test_together_rerank_async_honors_env_api_base(respx_mock: respx.MockRouter, monkeypatch): """Regression: TOGETHER_AI_API_BASE was honored by chat but ignored by rerank.""" From 1bf9418917af093d4d3d66885e1c5baa956995cb Mon Sep 17 00:00:00 2001 From: Wolfram Ravenwolf Date: Wed, 2 Sep 2026 02:48:16 +0200 Subject: [PATCH 024/411] fix(wandb): type supported parameter list --- litellm/llms/wandb/chat/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/wandb/chat/transformation.py b/litellm/llms/wandb/chat/transformation.py index cb9ac7b4c9b..9c3ff818331 100644 --- a/litellm/llms/wandb/chat/transformation.py +++ b/litellm/llms/wandb/chat/transformation.py @@ -10,7 +10,7 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig class WandbConfig(OpenAIGPTConfig): - def get_supported_openai_params(self, model: str) -> list: # mutable-ok: inherited contract + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract return super().get_supported_openai_params(model) + ["reasoning_effort"] # mutable-ok: inherited contract def map_openai_params( From b0aafabdeeb2b126b3e268a621b68c1ae5a32472 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:30:46 -0700 Subject: [PATCH 025/411] ci(lint): gate top-level tests/e2e and litellm files in the diff-scoped lint steps Without :(glob), git matches 'tests/e2e/**/*.py' with * crossing slashes, so the pattern needs at least one directory below tests/e2e and a top-level file never matches. PR #39209 added tests/e2e/test_junit_properties.py with three basedpyright errors and the e2e step printed "No changed tests/e2e Python files; skipping." The ruff format step's 'litellm/**/*.py' skipped litellm/main.py and the other top-level modules the same way. :(glob) makes /**/ match zero or more directories, so both gates now select top-level and nested files alike. A regression test runs the workflow's own pathspecs against a throwaway repo and locks that in for every diff-scoped gate. --- .github/workflows/test-linting.yml | 4 +- Makefile | 4 +- .../test_lint_workflow_diff_gates.py | 54 +++++++++++++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 tests/test_litellm/test_lint_workflow_diff_gates.py diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 9f1283da19e..06d369eabcd 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -113,7 +113,7 @@ jobs: - name: Check ruff format if: steps.changes.outputs.decision != 'skip' run: | - git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true + git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- ':(glob)litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then echo "No changed litellm Python files to check with ruff format." exit 0 @@ -172,7 +172,7 @@ jobs: - name: Check tests/e2e basedpyright (zero errors) if: steps.changes.outputs.decision != 'skip' run: | - if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then + if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' | grep -q .; then uv run --no-sync basedpyright tests/e2e else echo "No changed tests/e2e Python files; skipping." diff --git a/Makefile b/Makefile index e17fdba3c85..ea15e7da8ab 100644 --- a/Makefile +++ b/Makefile @@ -147,8 +147,8 @@ lint-install: # Diff-scoped format check, mirroring test-linting.yml's "Check ruff format" step: # only the litellm Python files changed vs the base are checked, so a pre-existing # format issue elsewhere doesn't block an unrelated commit. Git pathspecs match -# recursively, so 'litellm/*.py' covers nested modules and the top-level files that -# CI's 'litellm/**/*.py' skips, which makes this target a superset of the CI step. +# recursively, so 'litellm/*.py' covers top-level files and nested modules alike, +# the same set CI's ':(glob)litellm/**/*.py' selects. lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) @files=$$(git diff --name-only --diff-filter=ACMR origin/litellm_internal_staging...HEAD -- 'litellm/*.py' | grep -v '^litellm/enterprise/' || true); \ if [ -z "$$files" ]; then \ diff --git a/tests/test_litellm/test_lint_workflow_diff_gates.py b/tests/test_litellm/test_lint_workflow_diff_gates.py new file mode 100644 index 00000000000..62e67cdaaab --- /dev/null +++ b/tests/test_litellm/test_lint_workflow_diff_gates.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import re +import shlex +import subprocess +from pathlib import Path +from typing import Final + +import pytest + +WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-linting.yml" +DIFF_GATE: Final = re.compile(r'git diff --name-only --diff-filter=\w+ "\$GATE_BASE_SHA" HEAD -- (.+?) \|') +GATES: Final = tuple(tuple(shlex.split(gate.group(1))) for gate in DIFF_GATE.finditer(WORKFLOW.read_text())) + + +def _git(cwd: Path, *args: str) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout + + +def _scoped_root(pathspec: str) -> str: + return re.sub(r"^:\([^)]*\)", "", pathspec).split("*", 1)[0] + + +def _changed_files_selected_by(tmp_path: Path, pathspecs: tuple[str, ...], files: tuple[str, ...]) -> frozenset[str]: + _git(tmp_path, "init", "-q", "-b", "main") + _git(tmp_path, "config", "user.email", "t@t") + _git(tmp_path, "config", "user.name", "t") + _git(tmp_path, "commit", "-q", "--allow-empty", "-m", "base") + for name in files: + target = tmp_path / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("x = 1\n") + _git(tmp_path, "add", "-A") + _git(tmp_path, "commit", "-qm", "change") + return frozenset( + _git(tmp_path, "diff", "--name-only", "--diff-filter=ACMRD", "HEAD~1", "HEAD", "--", *pathspecs).split() + ) + + +def test_workflow_still_carries_the_ruff_format_and_e2e_basedpyright_diff_gates() -> None: + assert frozenset(_scoped_root(gate[0]) for gate in GATES) == frozenset({"litellm/", "tests/e2e/"}) + + +@pytest.mark.parametrize("pathspecs", GATES, ids=" ".join) +def test_diff_gate_selects_top_level_and_nested_python_files_only(tmp_path: Path, pathspecs: tuple[str, ...]) -> None: + root = _scoped_root(pathspecs[0]) + top_level = f"{root}top_level_module.py" + nested = f"{root}pkg/sub/nested_module.py" + selected = _changed_files_selected_by( + tmp_path, + pathspecs, + (top_level, nested, f"{root}notes.md", "elsewhere/top_level_module.py", "elsewhere/pkg/nested_module.py"), + ) + assert selected == frozenset({top_level, nested}) From 9d6b8bb14af003f0e8e3e4198e11efa434fbf2ab Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:33:16 -0700 Subject: [PATCH 026/411] test(e2e): throwaway top-level probe with a type error to prove the gate now runs --- tests/e2e/zz_pathspec_probe.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/e2e/zz_pathspec_probe.py diff --git a/tests/e2e/zz_pathspec_probe.py b/tests/e2e/zz_pathspec_probe.py new file mode 100644 index 00000000000..379502b5a0b --- /dev/null +++ b/tests/e2e/zz_pathspec_probe.py @@ -0,0 +1,3 @@ +from typing import Final + +PROBE: Final[int] = "a string where the gate must see an int" From eb49135f2a91887fa35e5c9dc820e643b2458034 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:39:47 -0700 Subject: [PATCH 027/411] test(e2e): remove the throwaway pathspec probe --- tests/e2e/zz_pathspec_probe.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 tests/e2e/zz_pathspec_probe.py diff --git a/tests/e2e/zz_pathspec_probe.py b/tests/e2e/zz_pathspec_probe.py deleted file mode 100644 index 379502b5a0b..00000000000 --- a/tests/e2e/zz_pathspec_probe.py +++ /dev/null @@ -1,3 +0,0 @@ -from typing import Final - -PROBE: Final[int] = "a string where the gate must see an int" From c60c60e6fe4a860377d9fcd9e611d71234b91e36 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 2 Sep 2026 10:35:49 +0200 Subject: [PATCH 028/411] fix(proxy): resolve /v1/models limits from the deployment, not the alias --- litellm/proxy/utils.py | 108 +++++++++++---- litellm/router.py | 42 ++++-- litellm/types/router.py | 17 +++ litellm/utils.py | 27 ++-- .../test_model_management_endpoints.py | 4 +- .../proxy/proxy_server/test_routes_models.py | 8 +- .../test_team_model_name_translation.py | 12 +- tests/test_litellm/proxy/test_proxy_utils.py | 127 +++++++++++++++++- tests/test_litellm/test_router.py | 99 ++++++++++++++ 9 files changed, 388 insertions(+), 56 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 051d36c4d0f..b86e6d857cc 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7481,6 +7481,64 @@ async def get_available_models_for_user( return all_models +def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo]) -> ModelInfo | None: + try: + return get_model_info(model) + except Exception as e: + verbose_proxy_logger.debug( + "create_model_info_response: cost map lookup failed for %s: %s", + model, + e, + ) + return None + + +def _resolve_listing_model_info( + deployment_model: str | None, + listed_model: str, + get_model_info: Callable[[str], ModelInfo], +) -> tuple[ModelInfo, ...]: + """ + Cost-map entries describing a listed model, best source first. + + The name a model is listed under is an arbitrary public alias, so it often misses the + cost map and lands on a fallback-generalization rule that answers with a conservative + family baseline instead of the real model's limits; the deployment's underlying model + is what the request actually reaches. Both names are kept because either can + generalize, and because a deployment's own model is registered into the cost map as a + stub that carries no limits of its own. Exact entries are consulted before generalized + ones, and each field is then taken from the first entry that has it. + """ + listed_info: Final = _safe_get_model_info(listed_model, get_model_info) + + # Fast path, and the only one a wildcard-expanded name takes: with a single name + # there is nothing to order, so skip the generalization test entirely. This keeps + # the per-model cost of the listing on the hot path #33721 exists to protect. + if deployment_model is None or deployment_model == listed_model: + return () if listed_info is None else (listed_info,) + + deployment_info: Final = _safe_get_model_info(deployment_model, get_model_info) + if deployment_info is None: + return () if listed_info is None else (listed_info,) + if listed_info is None: + return (deployment_info,) + + from litellm.utils import is_generalized_model_info + + # Both names resolved: the deployment's model leads unless it only generalized + # while the listed name is an exact cost-map entry. + if is_generalized_model_info(deployment_info) and not is_generalized_model_info(listed_info): + return (listed_info, deployment_info) + return (deployment_info, listed_info) + + +def _first_token_limit(candidates: tuple[ModelInfo, ...], field: str) -> int | None: + return next( + (limit for limit in (coerce_token_limit(info.get(field)) for info in candidates) if limit is not None), + None, + ) + + def create_model_info_response( model_id: str, provider: str, @@ -7505,31 +7563,35 @@ def create_model_info_response( "owned_by": provider, } - try: - model_cost_info: ModelInfo | None = get_model_info(model_id) - except Exception as e: - verbose_proxy_logger.debug( - "create_model_info_response: cost map lookup failed for %s: %s", - model_id, - e, - ) - model_cost_info = None + listing_info: Final = llm_router.get_model_listing_info(model_id) if llm_router is not None else None - max_input_tokens: int | None = None - max_output_tokens: int | None = None - if model_cost_info is not None: - max_input_tokens = coerce_token_limit(model_cost_info.get("max_input_tokens")) - max_output_tokens = coerce_token_limit(model_cost_info.get("max_output_tokens")) - mode: Final = model_cost_info.get("mode") - if isinstance(mode, str): - base["mode"] = mode + candidates: Final = _resolve_listing_model_info( + deployment_model=listing_info.cost_map_key if listing_info is not None else None, + listed_model=model_id, + get_model_info=get_model_info, + ) - if llm_router is not None: - configured_input, configured_output = llm_router.get_configured_token_limits(model_id) - if configured_input is not None: - max_input_tokens = configured_input - if configured_output is not None: - max_output_tokens = configured_output + max_input_tokens: int | None = _first_token_limit(candidates, "max_input_tokens") + max_output_tokens: int | None = _first_token_limit(candidates, "max_output_tokens") + mode: Final = next( + ( + m + for m in ( + cast("Mapping[str, object]", info).get("mode") # cast-ok: an entry need not carry "mode" + for info in candidates + ) + if isinstance(m, str) + ), + None, + ) + if mode is not None: + base["mode"] = mode + + if listing_info is not None: + if listing_info.max_input_tokens is not None: + max_input_tokens = listing_info.max_input_tokens + if listing_info.max_output_tokens is not None: + max_output_tokens = listing_info.max_output_tokens if max_input_tokens is not None: base["max_input_tokens"] = max_input_tokens diff --git a/litellm/router.py b/litellm/router.py index 23d8907fb49..7bc6a6c714a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -200,6 +200,7 @@ from litellm.types.router import ( CredentialLiteLLMParams, CustomRoutingStrategyBase, Deployment, + DeploymentModelListingInfo, DeploymentTypedDict, FallbackAccessCheck, GuardrailTypedDict, @@ -9726,25 +9727,44 @@ class Router: return None return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable + def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: + """ + Return what the concrete deployment behind model_name contributes to its + /v1/models entry: the cost-map key for its underlying model, plus any token + limits explicitly configured in its model_info. Resolved via O(1) index lookup. + + Returns None for wildcard-expanded or unknown names, where the listed name is + the real model name and no deployment-specific information exists, and treats a + malformed configured limit as absent rather than failing the listing. Unlike + get_model_group_info, this never triggers pattern matching or deep copies, so it + is safe to call per listed model on the /v1/models hot path. + """ + deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + + model_info: Final = deployment.model_info + # base_model is a declared field, so read it as one: an unset or blank value + # means the deployment's own model name is the cost-map key. + base_model: Final = model_info.base_model + return DeploymentModelListingInfo( + cost_map_key=base_model or deployment.litellm_params.model, + max_input_tokens=coerce_token_limit(model_info.get("max_input_tokens")), + max_output_tokens=coerce_token_limit(model_info.get("max_output_tokens")), + ) + def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete deployment's model_info for model_name, via O(1) index lookup. Returns (None, None) for wildcard-expanded or unknown names, and treats a - malformed configured value as absent rather than failing the listing. Unlike - get_model_group_info, this never triggers pattern matching or deep copies, so it - is safe to call per listed model on the /v1/models hot path. + malformed configured value as absent rather than failing the caller. """ - deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) - if deployment is None: + listing_info: Final = self.get_model_listing_info(model_name=model_name) + if listing_info is None: return (None, None) - - model_info: Final = deployment.model_info - return ( - coerce_token_limit(model_info.get("max_input_tokens")), - coerce_token_limit(model_info.get("max_output_tokens")), - ) + return (listing_info.max_input_tokens, listing_info.max_output_tokens) def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index e0957383aac..6f9fb4bf6ee 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -575,6 +575,23 @@ class Deployment(BaseModel): setattr(self, key, value) +@dataclass(frozen=True, slots=True) +class DeploymentModelListingInfo: + """What a concrete deployment contributes to its OpenAI-compatible listing entry. + + ``cost_map_key`` is the name the deployment's underlying model is known by in + ``litellm.model_cost`` (``model_info.base_model`` when set, else + ``litellm_params.model``), which is what the request actually reaches; the public + model name it is listed under is an arbitrary alias and often absent from the cost + map. The token limits are the ones explicitly set in ``model_info``, which outrank + anything the cost map says. + """ + + cost_map_key: str + max_input_tokens: int | None + max_output_tokens: int | None + + class RouterErrors(enum.Enum): """ Enum for router specific errors with common codes diff --git a/litellm/utils.py b/litellm/utils.py index 252b6756937..80d2399c518 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2948,24 +2948,33 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> dict[str, obje return None +def is_generalized_model_info(model_info: ModelInfo) -> bool: + """Whether ``model_info`` came from a fallback-generalization capability rule. + + Detected as the resolved key missing ``litellm.model_cost`` while matching a + capability rule. A rule-derived entry carries no pricing and only a conservative + family-baseline context window, so callers holding a second candidate name should + prefer an exact cost-map entry from that name over this one. + """ + key: Final = cast("Mapping[str, object]", model_info).get("key") # cast-ok: partial dicts may omit "key" + if not isinstance(key, str): + return False + return key not in litellm.model_cost and match_capability_generalizations(key) is not None + + def _get_builtin_model_info_for_registration(model: str) -> ModelInfo | None: """Resolve ``model`` to its built-in cost-map entry for registration merging. Returns ``None`` when the lookup raises or when it resolved via a - fallback-generalization capability rule, detected as the resolved key missing - ``litellm.model_cost`` while matching a capability rule. A rule-derived entry - carries no pricing, so treating it as a hit would skip the built-in - cache-pricing inheritance for prefix-mangled keys. + fallback-generalization capability rule. A rule-derived entry carries no + pricing, so treating it as a hit would skip the built-in cache-pricing + inheritance for prefix-mangled keys. """ try: info: Final = get_model_info(model=model) except Exception: return None - if info["key"] in litellm.model_cost: - return info - if match_capability_generalizations(info["key"]) is None: - return info - return None + return None if is_generalized_model_info(info) else info _runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 4661cc17dbc..55e195ff14b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -1944,7 +1944,7 @@ class TestModelInfoEndpoint: ): mock_router.get_fully_blocked_model_names.return_value = set() mock_router.get_model_list.return_value = [] - mock_router.get_configured_token_limits.return_value = (None, None) + mock_router.get_model_listing_info.return_value = None mock_router.get_deployment_by_model_group_name.return_value = Deployment( model_name="gpt-4", litellm_params=LiteLLM_Params(model="openai/gpt-4"), @@ -2021,7 +2021,7 @@ class TestModelInfoEndpoint: ): mock_router.get_fully_blocked_model_names.return_value = set() mock_router.get_model_list.return_value = [] - mock_router.get_configured_token_limits.return_value = (None, None) + mock_router.get_model_listing_info.return_value = None mock_router.get_deployment_by_model_group_name.return_value = Deployment( model_name="team-model-1", litellm_params=LiteLLM_Params(model="custom/team-model-1"), diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index 2b126b1ea95..9063010b366 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -17,6 +17,7 @@ import litellm from litellm.proxy import proxy_server from litellm.proxy import utils as proxy_utils from litellm.proxy.utils import create_model_info_response +from litellm.types.router import DeploymentModelListingInfo from .conftest import normalize # type: ignore[import-not-found] @@ -165,13 +166,16 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as built from another entry's lookup shows up as the wrong numbers.""" def _configured(model_name): - return (300000, 32000) if model_name == "gpt-4" else (500000, 4096) + max_input, max_output = (300000, 32000) if model_name == "gpt-4" else (500000, 4096) + return DeploymentModelListingInfo( + cost_map_key=model_name, max_input_tokens=max_input, max_output_tokens=max_output + ) def _cost_map_lookup(model_id): max_input, max_output = (200000, 64000) if model_id == "gpt-4" else (100000, 8000) return {"max_input_tokens": max_input, "max_output_tokens": max_output, "mode": "chat"} - patched_models.get_configured_token_limits = MagicMock(side_effect=_configured) + patched_models.get_model_listing_info = MagicMock(side_effect=_configured) def _resolved(**kwargs): return create_model_info_response(**kwargs, get_model_info=_cost_map_lookup) diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index aa35fd64f18..6c1f2eb9bcf 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -877,7 +877,7 @@ async def test_v1_models_translates_team_model_for_access_group_key(monkeypatch) router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_dep] router.get_model_list.return_value = [team_dep] @@ -919,7 +919,7 @@ async def test_v1_models_keeps_internal_names_when_public_name_flag_disabled( router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_dep] router.get_model_list.return_value = [team_dep] @@ -954,7 +954,7 @@ async def test_v1_models_translates_team_model_with_metadata(monkeypatch): router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_dep] router.get_model_list.return_value = [team_dep] router.get_model_group_info.return_value = None @@ -1000,7 +1000,7 @@ async def test_v1_models_metadata_fallbacks_use_internal_routing_key(monkeypatch router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_dep] router.get_model_list.return_value = [team_dep] # Fallbacks are keyed on the internal routing name, as the router stores them. @@ -1057,7 +1057,7 @@ async def test_v1_models_metadata_does_not_leak_other_team_fallbacks(monkeypatch router.get_model_names.return_value = ["model_name_teamX_uuid9"] router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} router.get_fully_blocked_model_names.return_value = set() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None router.model_list = [team_x, team_y] router.get_model_list.return_value = [team_x, team_y] router.fallbacks = [ @@ -1312,7 +1312,7 @@ def test_translate_team_model_names_for_listing_respects_legacy_flag(): def _public_named_router(*team_rows: dict) -> MagicMock: router = MagicMock() router.get_model_list.return_value = list(team_rows) - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None return router diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index dcaad968663..58f62bc4928 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -886,6 +886,7 @@ from typing import cast import litellm from litellm.proxy.utils import create_model_info_response +from litellm.types.router import DeploymentModelListingInfo from litellm.types.utils import ModelInfo @@ -913,7 +914,7 @@ def test_create_model_info_response_includes_max_tokens_from_lookup(): def test_create_model_info_response_does_not_call_router_group_info(): router = MagicMock() - router.get_configured_token_limits.return_value = (None, None) + router.get_model_listing_info.return_value = None response = create_model_info_response( model_id="some-model", @@ -928,7 +929,9 @@ def test_create_model_info_response_does_not_call_router_group_info(): def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map(): router = MagicMock() - router.get_configured_token_limits.return_value = (32000, 8000) + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_key="my-custom-deployment", max_input_tokens=32000, max_output_tokens=8000 + ) response = create_model_info_response( model_id="my-custom-deployment", @@ -944,7 +947,9 @@ def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map( def test_create_model_info_response_deployment_limits_override_cost_map(): router = MagicMock() - router.get_configured_token_limits.return_value = (200000, None) + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_key="gpt-4o", max_input_tokens=200000, max_output_tokens=None + ) response = create_model_info_response( model_id="gpt-4o", @@ -1878,3 +1883,119 @@ async def test_proxy_only_error_5xx_keeps_traceback_and_runs_sync_callbacks(monk Logging.failure_handler = orig_sync_failure assert "test_proxy_utils" in captured["async_traceback"] + + +def test_create_model_info_response_resolves_alias_to_deployment_model(): + """A public model name that is not itself a cost-map key must not be resolved through + the fallback-generalization rules: `bedrock-claude-opus-5` matches the generic + claude-family baseline (200k/64k) by substring, while the deployment it fronts really + accepts 1M/128k. Regression for the /v1/models alias resolution introduced in v1.94.0.""" + from litellm import Router + + saved_model_cost = dict(litellm.model_cost) + try: + router = Router( + model_list=[ + { + "model_name": "bedrock-claude-opus-5", + "litellm_params": { + "custom_llm_provider": "bedrock", + "model": "bedrock/eu.anthropic.claude-opus-5", + }, + "model_info": {"base_model": "eu.anthropic.claude-opus-5"}, + } + ] + ) + + response = create_model_info_response( + model_id="bedrock-claude-opus-5", provider="openai", llm_router=router + ) + finally: + litellm.model_cost.clear() + litellm.model_cost.update(saved_model_cost) + + assert response["max_input_tokens"] == 1000000 + assert response["max_output_tokens"] == 128000 + + +def test_create_model_info_response_keeps_exact_alias_over_generalized_deployment_model(): + """Mirror of the alias bug: when the deployment points at a custom backend name that + only matches a generalization rule, the listed name's exact cost-map entry is the + better answer and must win.""" + from litellm import Router + + saved_model_cost = dict(litellm.model_cost) + try: + router = Router( + model_list=[ + { + "model_name": "claude-opus-5", + "litellm_params": { + "custom_llm_provider": "bedrock", + "model": "bedrock/my-claude-opus-5-provisioned", + }, + } + ] + ) + + response = create_model_info_response( + model_id="claude-opus-5", provider="openai", llm_router=router + ) + finally: + litellm.model_cost.clear() + litellm.model_cost.update(saved_model_cost) + + assert response["max_input_tokens"] == 1000000 + + +def test_create_model_info_response_falls_back_to_alias_for_opaque_deployment_name(): + """An Azure deployment named after the resource rather than the model has no cost-map + entry; the listed name still does, and must keep answering.""" + from litellm import Router + + saved_model_cost = dict(litellm.model_cost) + try: + router = Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "azure/my-gpt4o-deployment"}, + } + ] + ) + + response = create_model_info_response( + model_id="gpt-4o", provider="openai", llm_router=router + ) + finally: + litellm.model_cost.clear() + litellm.model_cost.update(saved_model_cost) + + assert response["max_input_tokens"] == 128000 + assert response["max_output_tokens"] == 16384 + + +def test_create_model_info_response_resolves_mode_through_deployment_model(): + """`mode` is derived from the same lookup, so an aliased embedding deployment + currently reports no mode at all; it must report `embedding`.""" + from litellm import Router + + saved_model_cost = dict(litellm.model_cost) + try: + router = Router( + model_list=[ + { + "model_name": "my-embeddings", + "litellm_params": {"model": "openai/text-embedding-3-small"}, + } + ] + ) + + response = create_model_info_response( + model_id="my-embeddings", provider="openai", llm_router=router + ) + finally: + litellm.model_cost.clear() + litellm.model_cost.update(saved_model_cost) + + assert response["mode"] == "embedding" diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 84f6344be35..37805436f8d 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7271,6 +7271,105 @@ def test_get_configured_token_limits_coerces_numeric_strings(): assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000) +def test_get_model_listing_info_prefers_base_model_over_litellm_params_model(): + """The cost-map key comes from base_model when set, so a deployment pointing at an + opaque backend name still resolves the real catalog entry.""" + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-claude-opus-5", + "litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"}, + "model_info": {"base_model": "eu.anthropic.claude-opus-5"}, + } + ] + ) + + info = router.get_model_listing_info("bedrock-claude-opus-5") + assert info is not None + assert info.cost_map_key == "eu.anthropic.claude-opus-5" + + +def test_get_model_listing_info_falls_back_to_litellm_params_model(): + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-claude-opus-5", + "litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"}, + } + ] + ) + + info = router.get_model_listing_info("bedrock-claude-opus-5") + assert info is not None + assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5" + + +def test_get_model_listing_info_ignores_blank_base_model(): + """A base_model set to an empty string is absent, not a cost-map key.""" + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-claude-opus-5", + "litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"}, + "model_info": {"base_model": ""}, + } + ] + ) + + info = router.get_model_listing_info("bedrock-claude-opus-5") + assert info is not None + assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5" + + +def test_get_model_listing_info_returns_none_for_unknown_name(): + router = litellm.Router( + model_list=[ + { + "model_name": "no-limits-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + } + ] + ) + + assert router.get_model_listing_info("not-a-real-model") is None + + +def test_get_model_listing_info_carries_configured_limits(): + router = litellm.Router( + model_list=[ + { + "model_name": "my-custom-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": 32000, "max_output_tokens": 8000}, + } + ] + ) + + info = router.get_model_listing_info("my-custom-model") + assert info is not None + assert (info.max_input_tokens, info.max_output_tokens) == (32000, 8000) + + +def test_get_model_listing_info_skips_wildcard_pattern_matching(): + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock/*", + "litellm_params": {"model": "bedrock/*"}, + "model_info": {"max_input_tokens": 12345}, + } + ] + ) + + with patch.object( + router.pattern_router, "route", side_effect=AssertionError("pattern route called") + ): + assert ( + router.get_model_listing_info("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") + is None + ) + + @pytest.mark.asyncio async def test_acreate_batch_disable_fallbacks_surfaces_owning_provider_error(): router = litellm.Router( From d5270890c409fe5942ff98c5b3a69370fc0bff81 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 2 Sep 2026 12:03:36 +0200 Subject: [PATCH 029/411] fix(proxy): report the widest window across a model group, not the first deployment's --- litellm/proxy/utils.py | 45 +++++++-- litellm/router.py | 79 +++++++++++----- litellm/types/router.py | 17 ++-- .../proxy/proxy_server/test_routes_models.py | 2 +- tests/test_litellm/proxy/test_proxy_utils.py | 52 ++++++++++- tests/test_litellm/test_router.py | 91 ++++++++++++++++++- 6 files changed, 242 insertions(+), 44 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b86e6d857cc..874dca4238f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7496,10 +7496,11 @@ def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo]) def _resolve_listing_model_info( deployment_model: str | None, listed_model: str, + listed_info: ModelInfo | None, get_model_info: Callable[[str], ModelInfo], ) -> tuple[ModelInfo, ...]: """ - Cost-map entries describing a listed model, best source first. + Cost-map entries describing one deployment behind a listed model, best source first. The name a model is listed under is an arbitrary public alias, so it often misses the cost map and lands on a fallback-generalization rule that answers with a conservative @@ -7508,9 +7509,10 @@ def _resolve_listing_model_info( generalize, and because a deployment's own model is registered into the cost map as a stub that carries no limits of its own. Exact entries are consulted before generalized ones, and each field is then taken from the first entry that has it. - """ - listed_info: Final = _safe_get_model_info(listed_model, get_model_info) + ``listed_info`` is resolved once by the caller, since a group with several distinct + underlying models resolves the same alias for each of them. + """ # Fast path, and the only one a wildcard-expanded name takes: with a single name # there is nothing to order, so skip the generalization test entirely. This keeps # the per-model cost of the listing on the hot path #33721 exists to protect. @@ -7539,6 +7541,20 @@ def _first_token_limit(candidates: tuple[ModelInfo, ...], field: str) -> int | N ) +def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: str) -> int | None: + """The widest limit any deployment behind the listed name declares for ``field``. + + A model group is normally one model behind several interchangeable deployments, so + there is a single value to report. When a group genuinely mixes models, reporting the + widest window keeps the listing independent of config order and agreeing with + ``/model_group/info``, which aggregates the same way for the Admin UI. + """ + limits: Final = tuple( + limit for limit in (_first_token_limit(candidates, field) for candidates in candidate_sets) if limit is not None + ) + return max(limits) if limits else None + + def create_model_info_response( model_id: str, provider: str, @@ -7565,19 +7581,30 @@ def create_model_info_response( listing_info: Final = llm_router.get_model_listing_info(model_id) if llm_router is not None else None - candidates: Final = _resolve_listing_model_info( - deployment_model=listing_info.cost_map_key if listing_info is not None else None, - listed_model=model_id, - get_model_info=get_model_info, + # One entry per distinct model behind the listed name; (None,) when the router knows + # nothing about it, so the listed name is resolved on its own as before. + deployment_models: Final[tuple[str | None, ...]] = ( + listing_info.cost_map_keys if listing_info is not None and listing_info.cost_map_keys else (None,) + ) + listed_info: Final = _safe_get_model_info(model_id, get_model_info) + candidate_sets: Final = tuple( + _resolve_listing_model_info( + deployment_model=deployment_model, + listed_model=model_id, + listed_info=listed_info, + get_model_info=get_model_info, + ) + for deployment_model in deployment_models ) - max_input_tokens: int | None = _first_token_limit(candidates, "max_input_tokens") - max_output_tokens: int | None = _first_token_limit(candidates, "max_output_tokens") + max_input_tokens: int | None = _group_token_limit(candidate_sets, "max_input_tokens") + max_output_tokens: int | None = _group_token_limit(candidate_sets, "max_output_tokens") mode: Final = next( ( m for m in ( cast("Mapping[str, object]", info).get("mode") # cast-ok: an entry need not carry "mode" + for candidates in candidate_sets for info in candidates ) if isinstance(m, str) diff --git a/litellm/router.py b/litellm/router.py index 7bc6a6c714a..7011f7f504c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9729,29 +9729,57 @@ class Router: def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: """ - Return what the concrete deployment behind model_name contributes to its - /v1/models entry: the cost-map key for its underlying model, plus any token - limits explicitly configured in its model_info. Resolved via O(1) index lookup. + Return what the concrete deployments behind model_name contribute to its + /v1/models entry: the cost-map keys for their underlying models, plus the widest + token limits explicitly configured in their model_info. Resolved via O(1) index + lookup. - Returns None for wildcard-expanded or unknown names, where the listed name is - the real model name and no deployment-specific information exists, and treats a - malformed configured limit as absent rather than failing the listing. Unlike - get_model_group_info, this never triggers pattern matching or deep copies, so it - is safe to call per listed model on the /v1/models hot path. + Returns None for wildcard-expanded or unknown names, where the listed name is the + real model name and no deployment-specific information exists, and treats a + malformed configured limit as absent rather than failing the listing. + + The whole group is read rather than just its first deployment, so a group that + mixes models does not advertise a window that depends on config order; the widest + one is reported, which is what get_model_group_info already shows the Admin UI. + Keys are deduplicated, so the ordinary group of interchangeable deployments of one + model still costs the caller a single cost-map lookup. Unlike get_model_group_info, + this never triggers pattern matching or deep copies, so it is safe to call per + listed model on the /v1/models hot path. """ - deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) - if deployment is None: + indices: Final = self.model_name_to_deployment_indices.get(model_name) + if not indices: return None - model_info: Final = deployment.model_info - # base_model is a declared field, so read it as one: an unset or blank value - # means the deployment's own model name is the cost-map key. - base_model: Final = model_info.base_model - return DeploymentModelListingInfo( - cost_map_key=base_model or deployment.litellm_params.model, - max_input_tokens=coerce_token_limit(model_info.get("max_input_tokens")), - max_output_tokens=coerce_token_limit(model_info.get("max_output_tokens")), + deployments: Final = tuple(self.model_list[index] for index in indices) + model_infos: Final = tuple(deployment.get("model_info") or MappingProxyType({}) for deployment in deployments) + params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments) + # base_model resolution mirrors get_router_model_info: unset or blank means the + # deployment's own model name is the cost-map key. + cost_map_keys: Final = tuple( + dict.fromkeys( # deduplicates while preserving config order + key + for key in ( + model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") + for model_info, litellm_params in zip(model_infos, params) + ) + if isinstance(key, str) and key + ) ) + return DeploymentModelListingInfo( + cost_map_keys=cost_map_keys, + max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), + max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), + ) + + @staticmethod + def _widest_configured_limit(model_infos: Sequence[Mapping[str, Any]], field: str) -> int | None: + """The largest usable value of ``field`` across a group's configured model_info blocks.""" + limits: Final = tuple( + limit + for limit in (coerce_token_limit(model_info.get(field)) for model_info in model_infos) + if limit is not None + ) + return max(limits) if limits else None def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ @@ -9760,11 +9788,20 @@ class Router: Returns (None, None) for wildcard-expanded or unknown names, and treats a malformed configured value as absent rather than failing the caller. + + Deliberately reads one deployment rather than aggregating the group the way + get_model_listing_info does: its caller truncates an embedding input to this + value, so the widest window in a mixed group would be the wrong answer there. """ - listing_info: Final = self.get_model_listing_info(model_name=model_name) - if listing_info is None: + deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: return (None, None) - return (listing_info.max_input_tokens, listing_info.max_output_tokens) + + model_info: Final = deployment.model_info + return ( + coerce_token_limit(model_info.get("max_input_tokens")), + coerce_token_limit(model_info.get("max_output_tokens")), + ) def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 6f9fb4bf6ee..a773959f5f1 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -577,17 +577,18 @@ class Deployment(BaseModel): @dataclass(frozen=True, slots=True) class DeploymentModelListingInfo: - """What a concrete deployment contributes to its OpenAI-compatible listing entry. + """What the deployments behind a model name contribute to its OpenAI-compatible listing entry. - ``cost_map_key`` is the name the deployment's underlying model is known by in - ``litellm.model_cost`` (``model_info.base_model`` when set, else - ``litellm_params.model``), which is what the request actually reaches; the public - model name it is listed under is an arbitrary alias and often absent from the cost - map. The token limits are the ones explicitly set in ``model_info``, which outrank - anything the cost map says. + ``cost_map_keys`` are the names those deployments' underlying models are known by in + ``litellm.model_cost`` (``base_model`` when set, else ``litellm_params.model``), which + is what a request actually reaches; the public model name they are listed under is an + arbitrary alias and often absent from the cost map. Keys are deduplicated in config + order, so the ordinary group -- several interchangeable deployments of one model -- + carries exactly one. The token limits are the widest explicitly set in any + deployment's ``model_info``, which outrank anything the cost map says. """ - cost_map_key: str + cost_map_keys: tuple[str, ...] max_input_tokens: int | None max_output_tokens: int | None diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index 9063010b366..54768fef2ce 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -168,7 +168,7 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as def _configured(model_name): max_input, max_output = (300000, 32000) if model_name == "gpt-4" else (500000, 4096) return DeploymentModelListingInfo( - cost_map_key=model_name, max_input_tokens=max_input, max_output_tokens=max_output + cost_map_keys=(model_name,), max_input_tokens=max_input, max_output_tokens=max_output ) def _cost_map_lookup(model_id): diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 58f62bc4928..90e7431a22d 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -930,7 +930,7 @@ def test_create_model_info_response_does_not_call_router_group_info(): def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map(): router = MagicMock() router.get_model_listing_info.return_value = DeploymentModelListingInfo( - cost_map_key="my-custom-deployment", max_input_tokens=32000, max_output_tokens=8000 + cost_map_keys=("my-custom-deployment",), max_input_tokens=32000, max_output_tokens=8000 ) response = create_model_info_response( @@ -948,7 +948,7 @@ def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map( def test_create_model_info_response_deployment_limits_override_cost_map(): router = MagicMock() router.get_model_listing_info.return_value = DeploymentModelListingInfo( - cost_map_key="gpt-4o", max_input_tokens=200000, max_output_tokens=None + cost_map_keys=("gpt-4o",), max_input_tokens=200000, max_output_tokens=None ) response = create_model_info_response( @@ -962,6 +962,54 @@ def test_create_model_info_response_deployment_limits_override_cost_map(): assert response["max_output_tokens"] == 16384 +def test_create_model_info_response_reports_widest_window_in_a_mixed_group(): + """A group mixing models advertises the widest window, not whichever is listed first.""" + limits = { + "small-model": _fake_model_info(max_input_tokens=200000, max_output_tokens=4096, mode="chat"), + "large-model": _fake_model_info(max_input_tokens=1000000, max_output_tokens=128000, mode="chat"), + } + + for keys in (("small-model", "large-model"), ("large-model", "small-model")): + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=keys, max_input_tokens=None, max_output_tokens=None + ) + + response = create_model_info_response( + model_id="house-claude", + provider="openai", + llm_router=router, + get_model_info=lambda model: limits[model], + ) + + assert response["max_input_tokens"] == 1000000, keys + assert response["max_output_tokens"] == 128000, keys + + +def test_create_model_info_response_resolves_alias_once_per_listing(): + """The alias is the same for every deployment in the group, so it is looked up once.""" + seen: list[str] = [] + + def _tracking_get_model_info(model: str) -> ModelInfo: + seen.append(model) + return _fake_model_info(max_input_tokens=128000) + + router = MagicMock() + router.get_model_listing_info.return_value = DeploymentModelListingInfo( + cost_map_keys=("model-a", "model-b"), max_input_tokens=None, max_output_tokens=None + ) + + create_model_info_response( + model_id="house-model", + provider="openai", + llm_router=router, + get_model_info=_tracking_get_model_info, + ) + + assert seen.count("house-model") == 1 + assert sorted(seen) == ["house-model", "model-a", "model-b"] + + def test_create_model_info_response_survives_malformed_configured_limits(): from litellm import Router diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 37805436f8d..588fd98abd1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7286,7 +7286,7 @@ def test_get_model_listing_info_prefers_base_model_over_litellm_params_model(): info = router.get_model_listing_info("bedrock-claude-opus-5") assert info is not None - assert info.cost_map_key == "eu.anthropic.claude-opus-5" + assert info.cost_map_keys == ("eu.anthropic.claude-opus-5",) def test_get_model_listing_info_falls_back_to_litellm_params_model(): @@ -7301,7 +7301,7 @@ def test_get_model_listing_info_falls_back_to_litellm_params_model(): info = router.get_model_listing_info("bedrock-claude-opus-5") assert info is not None - assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5" + assert info.cost_map_keys == ("bedrock/eu.anthropic.claude-opus-5",) def test_get_model_listing_info_ignores_blank_base_model(): @@ -7318,7 +7318,7 @@ def test_get_model_listing_info_ignores_blank_base_model(): info = router.get_model_listing_info("bedrock-claude-opus-5") assert info is not None - assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5" + assert info.cost_map_keys == ("bedrock/eu.anthropic.claude-opus-5",) def test_get_model_listing_info_returns_none_for_unknown_name(): @@ -7350,6 +7350,91 @@ def test_get_model_listing_info_carries_configured_limits(): assert (info.max_input_tokens, info.max_output_tokens) == (32000, 8000) +def test_get_model_listing_info_dedupes_interchangeable_deployments(): + """The ordinary group is N deployments of one model, so it yields exactly one key.""" + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-a"}, + }, + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-b"}, + }, + ] + ) + + info = router.get_model_listing_info("gpt-4o") + assert info is not None + assert info.cost_map_keys == ("openai/gpt-4o",) + + +def test_get_model_listing_info_collects_every_model_in_a_mixed_group(): + router = litellm.Router( + model_list=[ + { + "model_name": "house-claude", + "litellm_params": {"model": "anthropic/claude-3-haiku-20240307"}, + }, + { + "model_name": "house-claude", + "litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"}, + }, + ] + ) + + info = router.get_model_listing_info("house-claude") + assert info is not None + assert info.cost_map_keys == ( + "anthropic/claude-3-haiku-20240307", + "bedrock/eu.anthropic.claude-opus-5", + ) + + +def test_get_model_listing_info_reports_widest_configured_limits_in_a_mixed_group(): + """Matches how get_model_group_info aggregates for the Admin UI, so the two agree.""" + router = litellm.Router( + model_list=[ + { + "model_name": "house-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"max_input_tokens": 32000, "max_output_tokens": 4096}, + }, + { + "model_name": "house-model", + "litellm_params": {"model": "openai/another-unmapped-model"}, + "model_info": {"max_input_tokens": 128000, "max_output_tokens": 16384}, + }, + ] + ) + + info = router.get_model_listing_info("house-model") + assert info is not None + assert (info.max_input_tokens, info.max_output_tokens) == (128000, 16384) + + +def test_get_model_listing_info_reads_base_model_from_litellm_params(): + """base_model resolution mirrors get_router_model_info, which also accepts it there.""" + router = litellm.Router( + model_list=[ + { + "model_name": "azure-deployment", + "litellm_params": { + "model": "azure/my-azure-deployment-name", + "base_model": "azure/gpt-4o", + "api_key": "sk-a", + "api_base": "https://example.openai.azure.com", + }, + } + ] + ) + + info = router.get_model_listing_info("azure-deployment") + assert info is not None + assert info.cost_map_keys == ("azure/gpt-4o",) + + def test_get_model_listing_info_skips_wildcard_pattern_matching(): router = litellm.Router( model_list=[ From f0f6ff8de4c5bd09b092206003f1f1aa00b7bb41 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 2 Sep 2026 14:12:23 +0200 Subject: [PATCH 030/411] test(router): cover _widest_configured_limit directly for the coverage gate --- litellm/proxy/utils.py | 14 +++++++++++--- tests/test_litellm/test_router.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 874dca4238f..f891c85320a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7545,9 +7545,17 @@ def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: """The widest limit any deployment behind the listed name declares for ``field``. A model group is normally one model behind several interchangeable deployments, so - there is a single value to report. When a group genuinely mixes models, reporting the - widest window keeps the listing independent of config order and agreeing with - ``/model_group/info``, which aggregates the same way for the Admin UI. + there is a single value to report and the choice of aggregate does not arise. + + When a group genuinely mixes models no single number is right, and the widest is the + deliberate pick over the narrowest for two reasons. It is what ``/model_group/info`` + has long reported to the Admin UI, so the two surfaces agree; disagreeing is the very + complaint this resolution path exists to fix. And of the two ways to be wrong, + under-advertising is worse: a client that trusts a narrowed window silently refuses + prompts the group would have served, while an over-long prompt that reaches a smaller + deployment comes back as a legible context-length error -- and does not reach one at + all when ``enable_pre_call_checks`` is set, which filters deployments the prompt does + not fit. """ limits: Final = tuple( limit for limit in (_first_token_limit(candidates, field) for candidates in candidate_sets) if limit is not None diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 588fd98abd1..eba9a85feb6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7350,6 +7350,20 @@ def test_get_model_listing_info_carries_configured_limits(): assert (info.max_input_tokens, info.max_output_tokens) == (32000, 8000) +def test_widest_configured_limit_ignores_absent_and_malformed_values(): + model_infos = ( + {"max_input_tokens": 32000}, + {}, + {"max_input_tokens": "not-a-number"}, + {"max_input_tokens": "128000"}, + {"max_output_tokens": 4096}, + ) + + assert litellm.Router._widest_configured_limit(model_infos, "max_input_tokens") == 128000 + assert litellm.Router._widest_configured_limit(model_infos, "max_output_tokens") == 4096 + assert litellm.Router._widest_configured_limit((), "max_input_tokens") is None + + def test_get_model_listing_info_dedupes_interchangeable_deployments(): """The ordinary group is N deployments of one model, so it yields exactly one key.""" router = litellm.Router( From 50874f9fc9cdc9e925d0fc7abda06a178a303980 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 15:52:47 +0000 Subject: [PATCH 031/411] fix(mcp): require audiences and prove refresh-token ownership for identity binding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/discoverable_endpoints.py | 9 ++ .../mcp_server/oauth_identity_binding.py | 93 +++++++++++--- .../types/mcp_server/mcp_server_manager.py | 4 +- .../mcp_server/test_oauth_identity_binding.py | 117 +++++++++++++++++- .../test_mcp_management_endpoints.py | 6 +- 5 files changed, 211 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 92128a82856..88f2c47f5fc 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -56,6 +56,8 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( revoke_refresh_token, ) from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + RefreshOwnershipProven, + RefreshTokenPresented, enforce_oauth_identity_binding, ) from litellm.proxy._experimental.mcp_server.oauth_utils import ( @@ -1046,7 +1048,13 @@ async def exchange_token_with_server( refresh_request_scope = scope or bridge_upstream_scope if refresh_request_scope: token_data["scope"] = refresh_request_scope + refresh_ownership = ( + RefreshOwnershipProven() + if bridge_upstream_refresh is not None + else RefreshTokenPresented(upstream_refresh_token) + ) else: + refresh_ownership = None if not code: raise HTTPException( status_code=400, @@ -1145,6 +1153,7 @@ async def exchange_token_with_server( token_response=token_response, litellm_user_id=resolved_user_id, grant_type=grant_type, + refresh_ownership=refresh_ownership, ) # Store server-side when the server is configured for per-user OAuth and diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index f132f59ae99..d4339c15e1a 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -14,6 +14,7 @@ from typing import Final, Literal import jwt from fastapi import HTTPException +from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache @@ -37,6 +38,7 @@ _jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) JwksFetcher = Callable[[MCPOAuthIdentityBinding], Awaitable[list[Mapping[str, object]]]] CallerPrincipalLoader = Callable[[str, MCPOAuthIdentityBinding], Awaitable[str | None]] +StoredRefreshTokenLoader = Callable[[str, str], Awaitable[str | None]] _RejectionCode = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] @@ -47,6 +49,19 @@ class _BindingRejection: description: str +@dataclass(frozen=True, slots=True) +class RefreshOwnershipProven: + """The gateway itself unwrapped the upstream refresh token from a sealed per-user envelope.""" + + +@dataclass(frozen=True, slots=True) +class RefreshTokenPresented: + refresh_token: str + + +RefreshOwnership = RefreshOwnershipProven | RefreshTokenPresented | None + + async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: jwks_url: Final[str] = binding.jwks_url or await _discover_jwks_url(binding.issuer) cached: Final = await _jwks_cache.async_get_cache(jwks_url) @@ -98,11 +113,8 @@ def _decode_id_token( signing_key.key, algorithms=list(_ALLOWED_ID_TOKEN_ALGORITHMS), issuer=binding.issuer, - audience=binding.audiences if binding.audiences else None, - options={ - "require": ["iss", "exp"], - "verify_aud": bool(binding.audiences), - }, + audience=binding.audiences, + options={"require": ["iss", "exp"]}, ) except jwt.InvalidTokenError as exc: return _BindingRejection( @@ -121,7 +133,11 @@ def _upstream_principal( code="oauth_identity_binding_failed", description=f"id_token has no usable '{binding.principal_claim}' claim", ) - if binding.principal_claim == "email" and binding.require_email_verified and claims.get("email_verified") is not True: + if ( + binding.principal_claim == "email" + and binding.require_email_verified + and claims.get("email_verified") is not True + ): return _BindingRejection( code="oauth_identity_binding_failed", description="id_token email is not verified (email_verified is not true)", @@ -142,6 +158,26 @@ async def _load_caller_principal(litellm_user_id: str, binding: MCPOAuthIdentity return loaded.user_email +async def _load_stored_refresh_token(litellm_user_id: str, server_id: str) -> str | None: + try: + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + get_user_oauth_credential, + ) + from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 + + prisma_client: Final = get_prisma_client_or_throw( + "Database not connected. Cannot verify OAuth refresh token ownership." + ) + cred: Final = await get_user_oauth_credential( + prisma_client=prisma_client, + user_id=litellm_user_id, + server_id=server_id, + ) + return cred.get("refresh_token") if cred else None + except Exception: # noqa: BLE001 # a credential lookup failure must fail closed + return None + + def _principals_match(upstream: str, caller: str, binding: MCPOAuthIdentityBinding) -> bool: if binding.principal_claim == "email" or binding.caller_field == "user_email": return upstream.strip().casefold() == caller.strip().casefold() @@ -153,17 +189,41 @@ async def _evaluate_binding( token_response: Mapping[str, object], litellm_user_id: str | None, grant_type: str, + server_id: str, + refresh_ownership: RefreshOwnership, jwks_fetcher: JwksFetcher, caller_principal_loader: CallerPrincipalLoader, + stored_refresh_token_loader: StoredRefreshTokenLoader, ) -> _BindingRejection | None: id_token: Final = token_response.get("id_token") if not isinstance(id_token, str) or not id_token: - if grant_type == "refresh_token": - return None - return _BindingRejection( - code="oauth_identity_binding_failed", - description="the upstream token response carries no id_token to bind the credential to a principal", - ) + if grant_type != "refresh_token": + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the upstream token response carries no id_token to bind the credential to a principal", + ) + match refresh_ownership: + case RefreshOwnershipProven(): + return None + case None: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="refresh_token grant without an id_token carries no refresh token to prove ownership of", + ) + case RefreshTokenPresented(refresh_token): + if not litellm_user_id: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the request carries no resolvable LiteLLM user identity to bind the credential to", + ) + stored: Final = await stored_refresh_token_loader(litellm_user_id, server_id) + if stored is None or stored != refresh_token: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the presented refresh_token is not the caller's stored credential for this server", + ) + return None + assert_never(refresh_ownership) if not litellm_user_id: return _BindingRejection( code="oauth_identity_binding_failed", @@ -204,15 +264,17 @@ async def enforce_oauth_identity_binding( token_response: Mapping[str, object], litellm_user_id: str | None, grant_type: str, + refresh_ownership: RefreshOwnership, jwks_fetcher: JwksFetcher = _fetch_issuer_jwks, caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, + stored_refresh_token_loader: StoredRefreshTokenLoader = _load_stored_refresh_token, ) -> None: """Validate the exchanged token's upstream principal against the LiteLLM caller. No-op when the server has no binding or it is disabled. In enforce mode a failure raises 403 before the caller returns, stores, or caches the token; in audit mode failures are logged only. - A refresh_token grant without an id_token is allowed in both modes: the stored credential keeps - the binding established at the original authorization_code exchange. + A refresh_token grant without an id_token is allowed only when the presented refresh token matches + the caller's stored credential or a sealed bridge envelope already proved ownership. """ binding: Final = server.oauth_identity_binding if binding is None or binding.mode == "disabled": @@ -222,8 +284,11 @@ async def enforce_oauth_identity_binding( token_response=token_response, litellm_user_id=litellm_user_id, grant_type=grant_type, + server_id=server.server_id, + refresh_ownership=refresh_ownership, jwks_fetcher=jwks_fetcher, caller_principal_loader=caller_principal_loader, + stored_refresh_token_loader=stored_refresh_token_loader, ) if rejection is None: return diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index cc3f6c8e8d8..cebc7decac4 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Any, Final, Literal -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from litellm.types.mcp import ( DEFAULT_SUBJECT_TOKEN_TYPE, @@ -52,7 +52,7 @@ class MCPOAuthIdentityBinding(BaseModel): mode: Literal["disabled", "audit", "enforce"] = "disabled" issuer: str jwks_url: str | None = None - audiences: list[str] = [] + audiences: list[str] = Field(min_length=1) principal_claim: str = "email" caller_field: Literal["user_email", "user_id"] = "user_email" require_email_verified: bool = True diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index 69eaabeda4a..de59020c44c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -7,8 +7,11 @@ import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa from fastapi import HTTPException +from pydantic import ValidationError from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + RefreshOwnershipProven, + RefreshTokenPresented, enforce_oauth_identity_binding, ) from litellm.types.mcp import MCPTransport @@ -54,6 +57,13 @@ def _caller_loader(email: str | None): return load +def _stored_refresh_token_loader(refresh_token: str | None): + async def load(_user_id: str, _server_id: str) -> str | None: + return refresh_token + + return load + + def _server(mode: str = "enforce", **binding_overrides: object) -> MCPServer: return MCPServer( server_id="srv-1", @@ -77,6 +87,7 @@ async def test_matching_principal_passes(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -92,6 +103,7 @@ async def test_mismatched_principal_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -108,6 +120,7 @@ async def test_missing_id_token_rejected_on_authorization_code(): token_response={"access_token": "at"}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -116,18 +129,89 @@ async def test_missing_id_token_rejected_on_authorization_code(): @pytest.mark.asyncio -async def test_refresh_without_id_token_allowed(): +async def test_refresh_without_id_token_allowed_when_presented_token_matches_stored_credential(): result: Final = await enforce_oauth_identity_binding( server=_server(), token_response={"access_token": "at"}, litellm_user_id="user-a", grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented("rt-1"), jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), ) assert result is None +@pytest.mark.asyncio +async def test_refresh_without_id_token_rejects_different_presented_token(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented("rt-stolen"), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_rejects_missing_stored_token(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented("rt-1"), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader(None), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_passes_when_bridge_proves_ownership(): + async def fail_if_called(_user_id: str, _server_id: str) -> str | None: + raise AssertionError("stored refresh token loader should not be called") + + result: Final = await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=RefreshOwnershipProven(), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=fail_if_called, + ) + assert result is None + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_rejects_without_ownership_proof(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + @pytest.mark.asyncio async def test_refresh_with_mismatched_id_token_rejected(): token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) @@ -137,6 +221,7 @@ async def test_refresh_with_mismatched_id_token_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="refresh_token", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -150,6 +235,7 @@ async def test_audit_mode_logs_but_does_not_reject(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -165,6 +251,7 @@ async def test_unverified_email_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -187,6 +274,7 @@ async def test_wrong_issuer_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -202,6 +290,7 @@ async def test_no_litellm_identity_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id=None, grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) @@ -215,6 +304,7 @@ async def test_disabled_binding_is_noop(): token_response={"access_token": "at"}, litellm_user_id=None, grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), ) @@ -234,7 +324,32 @@ async def test_no_binding_is_noop(): token_response={"access_token": "at"}, litellm_user_id=None, grant_type="authorization_code", + refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), ) assert result is None + + +def test_identity_binding_requires_non_empty_audiences(): + with pytest.raises(ValidationError): + MCPOAuthIdentityBinding(mode="enforce", issuer=ISSUER, audiences=[]) + with pytest.raises(ValidationError): + MCPOAuthIdentityBinding(mode="enforce", issuer=ISSUER) + + +@pytest.mark.asyncio +async def test_wrong_audience_rejected(): + token: Final = _sign_id_token({"aud": "other-client", "email": "alice@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index bfbbf816e69..f3510e7e0a8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -4674,7 +4674,11 @@ async def test_store_mcp_oauth_user_credential_blocked_when_identity_binding_enf name=server_id, url="https://mcp.example.com", transport=MCPTransport.http, - oauth_identity_binding=MCPOAuthIdentityBinding(mode="enforce", issuer="https://idp.example.com"), + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", + issuer="https://idp.example.com", + audiences=["litellm-client"], + ), ) store_mock = AsyncMock(return_value=None) From 35db36ee936643d36391eab1cb3c9f5031284b0d Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 16:04:53 +0000 Subject: [PATCH 032/411] fix(mcp): satisfy type discipline lint gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/discoverable_endpoints.py | 4 +- .../mcp_server/oauth_identity_binding.py | 39 ++++++++++++------- .../mcp_management_endpoints.py | 6 +-- .../types/mcp_server/mcp_server_manager.py | 2 +- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 88f2c47f5fc..5f54bbb4958 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1048,13 +1048,13 @@ async def exchange_token_with_server( refresh_request_scope = scope or bridge_upstream_scope if refresh_request_scope: token_data["scope"] = refresh_request_scope - refresh_ownership = ( + refresh_ownership = ( # rebind-ok: grant-specific branches assign one ownership value RefreshOwnershipProven() if bridge_upstream_refresh is not None else RefreshTokenPresented(upstream_refresh_token) ) else: - refresh_ownership = None + refresh_ownership = None # rebind-ok: grant-specific branches assign one ownership value if not code: raise HTTPException( status_code=400, diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index d4339c15e1a..b73227f3413 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -8,12 +8,13 @@ claim is compared to the caller's trusted LiteLLM identity. Mismatches fail clos and are logged in audit mode. """ -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass -from typing import Final, Literal +from typing import Final, Literal, TypeAlias import jwt from fastapi import HTTPException +from jwt.types import Options from typing_extensions import assert_never from litellm._logging import verbose_logger @@ -36,11 +37,20 @@ _ALLOWED_ID_TOKEN_ALGORITHMS: Final = ( _JWKS_CACHE_TTL_SECONDS: Final = 3600 _jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) -JwksFetcher = Callable[[MCPOAuthIdentityBinding], Awaitable[list[Mapping[str, object]]]] -CallerPrincipalLoader = Callable[[str, MCPOAuthIdentityBinding], Awaitable[str | None]] -StoredRefreshTokenLoader = Callable[[str, str], Awaitable[str | None]] +JwksFetcher: TypeAlias = Callable[ + [MCPOAuthIdentityBinding], # mutable-ok: Callable parameter syntax requires a list + Awaitable[Sequence[Mapping[str, object]]], +] +CallerPrincipalLoader: TypeAlias = Callable[ + [str, MCPOAuthIdentityBinding], # mutable-ok: Callable parameter syntax requires a list + Awaitable[str | None], +] +StoredRefreshTokenLoader: TypeAlias = Callable[ + [str, str], # mutable-ok: Callable parameter syntax requires a list + Awaitable[str | None], +] -_RejectionCode = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] +_RejectionCode: TypeAlias = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] @dataclass(frozen=True, slots=True) @@ -59,10 +69,10 @@ class RefreshTokenPresented: refresh_token: str -RefreshOwnership = RefreshOwnershipProven | RefreshTokenPresented | None +RefreshOwnership: TypeAlias = RefreshOwnershipProven | RefreshTokenPresented | None -async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: +async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mapping[str, object]]: jwks_url: Final[str] = binding.jwks_url or await _discover_jwks_url(binding.issuer) cached: Final = await _jwks_cache.async_get_cache(jwks_url) if isinstance(cached, list): @@ -90,12 +100,12 @@ async def _discover_jwks_url(issuer: str) -> str: return jwks_uri -def _select_signing_key(id_token: str, keys: list[Mapping[str, object]]) -> "jwt.PyJWK | _BindingRejection": +def _select_signing_key(id_token: str, keys: Sequence[Mapping[str, object]]) -> "jwt.PyJWK | _BindingRejection": header: Final = jwt.get_unverified_header(id_token) kid: Final = header.get("kid") for key in keys: if kid is None or key.get("kid") == kid: - return jwt.PyJWK(dict(key)) + return jwt.PyJWK(dict(key)) # mutable-ok: PyJWT requires a concrete JWK dictionary return _BindingRejection( code="oauth_identity_binding_failed", description=f"id_token signing key (kid={kid!r}) not found in the issuer's JWKS", @@ -108,13 +118,14 @@ def _decode_id_token( signing_key: "jwt.PyJWK", ) -> "Mapping[str, object] | _BindingRejection": try: + decode_options: Final[Options] = {"require": ("iss", "exp")} return jwt.decode( id_token, signing_key.key, - algorithms=list(_ALLOWED_ID_TOKEN_ALGORITHMS), + algorithms=_ALLOWED_ID_TOKEN_ALGORITHMS, issuer=binding.issuer, audience=binding.audiences, - options={"require": ["iss", "exp"]}, + options=decode_options, ) except jwt.InvalidTokenError as exc: return _BindingRejection( @@ -160,10 +171,10 @@ async def _load_caller_principal(litellm_user_id: str, binding: MCPOAuthIdentity async def _load_stored_refresh_token(litellm_user_id: str, server_id: str) -> str | None: try: - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # keep database imports lazy get_user_oauth_credential, ) - from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 + from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 # keep database imports lazy prisma_client: Final = get_prisma_client_or_throw( "Database not connected. Cannot verify OAuth refresh token ownership." diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 92a6997abff..109030a2df5 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -259,7 +259,7 @@ if MCP_AVAILABLE: if normalize_upstream_header_name(raw) is None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={ + detail={ # mutable-ok: FastAPI exception detail requires a JSON-serializable dictionary "error": ( f"Invalid upstream_token_header {raw!r}: must be a valid HTTP header name " "(RFC 7230 token, e.g. 'esb-oauth')" @@ -2256,7 +2256,7 @@ if MCP_AVAILABLE: """Persist the OAuth2 access token obtained by the calling user.""" prisma_client: Final = get_prisma_client_or_throw("Database not connected. Connect a database to your proxy") await _authorize_and_fetch_mcp_server(prisma_client, user_api_key_dict, server_id) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # keep manager import lazy global_mcp_server_manager as _manager, ) @@ -2267,7 +2267,7 @@ if MCP_AVAILABLE: if binding is not None and binding.mode == "enforce": raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail={ + detail={ # mutable-ok: FastAPI exception detail requires a JSON-serializable dictionary "error": "oauth_identity_binding_enforced", "error_description": ( "Direct credential storage is disabled for this server: its OAuth identity " diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index cebc7decac4..c38f1c366b1 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -52,7 +52,7 @@ class MCPOAuthIdentityBinding(BaseModel): mode: Literal["disabled", "audit", "enforce"] = "disabled" issuer: str jwks_url: str | None = None - audiences: list[str] = Field(min_length=1) + audiences: list[str] = Field(min_length=1) # mutable-ok: public Pydantic schema requires list values principal_claim: str = "email" caller_field: Literal["user_email", "user_id"] = "user_email" require_email_verified: bool = True From 22e3fcba54c2409a46e04ba81bd6885cbfc0442a Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 16:05:37 +0000 Subject: [PATCH 033/411] fix(mcp): restore unrelated exception formatting Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/management_endpoints/mcp_management_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 109030a2df5..fd6294fd77e 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -259,7 +259,7 @@ if MCP_AVAILABLE: if normalize_upstream_header_name(raw) is None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={ # mutable-ok: FastAPI exception detail requires a JSON-serializable dictionary + detail={ "error": ( f"Invalid upstream_token_header {raw!r}: must be a valid HTTP header name " "(RFC 7230 token, e.g. 'esb-oauth')" From 7ca9f28575021d026d28bb0bfb11a667ba3add77 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 16:33:59 +0000 Subject: [PATCH 034/411] test(mcp): cover OAuth identity binding paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/oauth_identity_binding.py | 2 +- .../mcp_server/test_discoverable_endpoints.py | 126 ++++++++++ .../mcp_server/test_oauth_identity_binding.py | 229 ++++++++++++++++++ 3 files changed, 356 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index b73227f3413..58ec3cfdb4f 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -234,7 +234,7 @@ async def _evaluate_binding( description="the presented refresh_token is not the caller's stored credential for this server", ) return None - assert_never(refresh_ownership) + assert_never(refresh_ownership) # pragma: no cover if not litellm_user_id: return _BindingRejection( code="oauth_identity_binding_failed", diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 598e9276423..4ae1b105dbf 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -7816,6 +7816,132 @@ async def test_token_exchange_pairs_client_secret_with_server_client_id(): assert "client_secret" not in sent +@pytest.mark.asyncio +async def test_token_exchange_refresh_passes_presented_refresh_ownership(): + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import RefreshTokenPresented + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + server = MCPServer( + server_id="srv-1", + name="srv-1", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="cid", + token_url="https://provider.example/token", + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", + issuer="https://provider.example", + audiences=["cid"], + ), + ) + request = MagicMock(spec=Request) + request.base_url = "https://litellm.example.com/" + request.headers = {} + response = MagicMock() + response.json.return_value = {"access_token": "at"} + response.raise_for_status = MagicMock() + client = MagicMock() + client.post = AsyncMock(return_value=response) + enforce = AsyncMock() + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=client, + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", + new=AsyncMock(return_value="user-a"), + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", + new=enforce, + ), + ): + await exchange_token_with_server( + request=request, + mcp_server=server, + grant_type="refresh_token", + code=None, + redirect_uri=None, + client_id="cid", + client_secret=None, + code_verifier=None, + refresh_token="rt-1", + ) + + ownership = enforce.await_args.kwargs["refresh_ownership"] + assert isinstance(ownership, RefreshTokenPresented) + assert ownership.refresh_token == "rt-1" + + +@pytest.mark.asyncio +async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + server = MCPServer( + server_id="srv-1", + name="srv-1", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="cid", + token_url="https://provider.example/token", + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", + issuer="https://provider.example", + audiences=["cid"], + ), + ) + request = MagicMock(spec=Request) + request.base_url = "https://litellm.example.com/" + request.headers = {} + response = MagicMock() + response.json.return_value = {"access_token": "at"} + response.raise_for_status = MagicMock() + client = MagicMock() + client.post = AsyncMock(return_value=response) + enforce = AsyncMock() + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=client, + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", + new=AsyncMock(return_value="user-a"), + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", + new=enforce, + ), + ): + await exchange_token_with_server( + request=request, + mcp_server=server, + grant_type="authorization_code", + code="auth-code", + redirect_uri="https://litellm.example.com/callback", + client_id="cid", + client_secret=None, + code_verifier=None, + ) + + assert enforce.await_args.kwargs["refresh_ownership"] is None + + def _upstream_token_response(status_code: int, *, json_body: object = None, text_body: str = "") -> "httpx.Response": import httpx diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index de59020c44c..c2662f9f298 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -1,6 +1,8 @@ import time from collections.abc import Mapping +from types import SimpleNamespace from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch import jwt import pytest @@ -12,6 +14,11 @@ from pydantic import ValidationError from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( RefreshOwnershipProven, RefreshTokenPresented, + _discover_jwks_url, + _fetch_issuer_jwks, + _load_caller_principal, + _load_stored_refresh_token, + _select_signing_key, enforce_oauth_identity_binding, ) from litellm.types.mcp import MCPTransport @@ -79,6 +86,121 @@ def _server(mode: str = "enforce", **binding_overrides: object) -> MCPServer: ) +@pytest.mark.asyncio +async def test_fetch_issuer_jwks_fetches_and_caches_keys(): + binding: Final = _server(jwks_url="https://idp.example.com/jwks-cache").oauth_identity_binding + assert binding is not None + response: Final = MagicMock() + response.json.return_value = {"keys": [_PUBLIC_JWK]} + client: Final = MagicMock() + client.get = AsyncMock(return_value=response) + + with patch( + "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + return_value=client, + ): + first: Final = await _fetch_issuer_jwks(binding) + second: Final = await _fetch_issuer_jwks(binding) + + assert first == second == [_PUBLIC_JWK] + client.get.assert_awaited_once_with("https://idp.example.com/jwks-cache") + + +@pytest.mark.asyncio +async def test_fetch_issuer_jwks_rejects_malformed_document(): + binding: Final = _server(jwks_url="https://idp.example.com/jwks-invalid").oauth_identity_binding + assert binding is not None + response: Final = MagicMock() + response.json.return_value = {} + client: Final = MagicMock() + client.get = AsyncMock(return_value=response) + + with patch( + "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + return_value=client, + ): + with pytest.raises(TypeError, match="has no 'keys' array"): + await _fetch_issuer_jwks(binding) + + +@pytest.mark.asyncio +async def test_discover_jwks_url_returns_provider_uri(): + response: Final = MagicMock() + response.json.return_value = {"jwks_uri": "https://idp.example.com/jwks"} + client: Final = MagicMock() + client.get = AsyncMock(return_value=response) + + with patch( + "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + return_value=client, + ): + result: Final = await _discover_jwks_url("https://idp.example.com/") + + assert result == "https://idp.example.com/jwks" + client.get.assert_awaited_once_with("https://idp.example.com/.well-known/openid-configuration") + + +@pytest.mark.asyncio +async def test_discover_jwks_url_rejects_missing_provider_uri(): + response: Final = MagicMock() + response.json.return_value = {} + client: Final = MagicMock() + client.get = AsyncMock(return_value=response) + + with patch( + "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + return_value=client, + ): + with pytest.raises(ValueError, match="returned no jwks_uri"): + await _discover_jwks_url("https://idp.example.com") + + +def test_select_signing_key_returns_matching_key_or_rejection(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + + selected: Final = _select_signing_key(token, [_PUBLIC_JWK]) + rejected: Final = _select_signing_key(token, []) + + assert selected.__class__.__name__ == "PyJWK" + assert rejected.code == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_load_caller_principal_supports_user_id_and_database_email(): + user_id_binding: Final = _server(caller_field="user_id").oauth_identity_binding + assert user_id_binding is not None + assert await _load_caller_principal("user-a", user_id_binding) == "user-a" + + with patch( + "litellm.proxy._experimental.mcp_server.bridge_token_flow.load_active_user_by_id", + new=AsyncMock(side_effect=["no_active_key", SimpleNamespace(user_email="alice@example.com")]), + ): + assert await _load_caller_principal("user-a", _server().oauth_identity_binding) is None + assert await _load_caller_principal("user-a", _server().oauth_identity_binding) == "alice@example.com" + + +@pytest.mark.asyncio +async def test_load_stored_refresh_token_returns_credential_and_fails_closed(): + get_credential: Final = AsyncMock(return_value={"refresh_token": "rt-1"}) + with ( + patch( + "litellm.proxy._experimental.mcp_server.db.get_user_oauth_credential", + new=get_credential, + ), + patch( + "litellm.proxy.utils.get_prisma_client_or_throw", + return_value="prisma", + ), + ): + assert await _load_stored_refresh_token("user-a", "srv-1") == "rt-1" + + with patch( + "litellm.proxy.utils.get_prisma_client_or_throw", + side_effect=RuntimeError("database unavailable"), + ): + assert await _load_stored_refresh_token("user-a", "srv-1") is None + + @pytest.mark.asyncio async def test_matching_principal_passes(): token: Final = _sign_id_token({"email": "Alice@Example.com", "email_verified": True}) @@ -112,6 +234,113 @@ async def test_mismatched_principal_rejected(): assert exc_info.value.detail["credential_stored"] is False +@pytest.mark.asyncio +async def test_missing_upstream_principal_rejected(): + token: Final = _sign_id_token({"email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "no usable" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_jwks_fetch_failure_is_rejected(): + async def fail(_binding: MCPOAuthIdentityBinding) -> list[Mapping[str, object]]: + raise RuntimeError("jwks unavailable") + + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=fail, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "jwks unavailable" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_missing_signing_key_is_rejected(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + + async def no_keys(_binding: MCPOAuthIdentityBinding) -> tuple[Mapping[str, object], ...]: + return () + + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=no_keys, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "signing key" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_missing_caller_principal_is_rejected(): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader(None), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "has no" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_refresh_without_id_token_requires_litellm_identity_for_presented_token(): + with pytest.raises(HTTPException) as exc_info: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id=None, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented("rt-1"), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), + ) + assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + assert "no resolvable LiteLLM user identity" in exc_info.value.detail["error_description"] + + +@pytest.mark.asyncio +async def test_user_id_principal_matching_uses_exact_comparison(): + token: Final = _sign_id_token({"sub": "user-a"}) + result: Final = await enforce_oauth_identity_binding( + server=_server(principal_claim="sub", caller_field="user_id"), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("user-a"), + ) + assert result is None + + @pytest.mark.asyncio async def test_missing_id_token_rejected_on_authorization_code(): with pytest.raises(HTTPException) as exc_info: From 540e528d2190e6ed7292174e1985673a602a3018 Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 16:43:18 +0000 Subject: [PATCH 035/411] test(mcp): satisfy test quality gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/test_discoverable_endpoints.py | 16 +++++++++------- .../mcp_server/test_oauth_identity_binding.py | 16 ++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 4ae1b105dbf..33b71f8d50f 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -7851,15 +7851,15 @@ async def test_token_exchange_refresh_passes_presented_refresh_ownership(): enforce = AsyncMock() with ( - patch( + patch( # test-quality-ok: no injection seam exists for the exchange's HTTP and identity collaborators "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=client, ), - patch( + patch( # test-quality-ok: no injection seam exists for request identity extraction "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", new=AsyncMock(return_value="user-a"), ), - patch( + patch( # test-quality-ok: captures the ownership value at the exchange boundary "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", new=enforce, ), @@ -7915,20 +7915,20 @@ async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): enforce = AsyncMock() with ( - patch( + patch( # test-quality-ok: no injection seam exists for the exchange's HTTP and identity collaborators "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=client, ), - patch( + patch( # test-quality-ok: no injection seam exists for request identity extraction "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", new=AsyncMock(return_value="user-a"), ), - patch( + patch( # test-quality-ok: captures the ownership value at the exchange boundary "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", new=enforce, ), ): - await exchange_token_with_server( + result = await exchange_token_with_server( request=request, mcp_server=server, grant_type="authorization_code", @@ -7939,6 +7939,8 @@ async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): code_verifier=None, ) + assert result.status_code == 200 + assert json.loads(result.body)["access_token"] == "at" assert enforce.await_args.kwargs["refresh_ownership"] is None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index c2662f9f298..36364c32ae5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -95,7 +95,7 @@ async def test_fetch_issuer_jwks_fetches_and_caches_keys(): client: Final = MagicMock() client.get = AsyncMock(return_value=response) - with patch( + with patch( # test-quality-ok: no HTTP dependency injection seam exists for JWKS fetching "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): @@ -115,7 +115,7 @@ async def test_fetch_issuer_jwks_rejects_malformed_document(): client: Final = MagicMock() client.get = AsyncMock(return_value=response) - with patch( + with patch( # test-quality-ok: no HTTP dependency injection seam exists for JWKS fetching "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): @@ -130,7 +130,7 @@ async def test_discover_jwks_url_returns_provider_uri(): client: Final = MagicMock() client.get = AsyncMock(return_value=response) - with patch( + with patch( # test-quality-ok: no HTTP dependency injection seam exists for OIDC discovery "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): @@ -147,7 +147,7 @@ async def test_discover_jwks_url_rejects_missing_provider_uri(): client: Final = MagicMock() client.get = AsyncMock(return_value=response) - with patch( + with patch( # test-quality-ok: no HTTP dependency injection seam exists for OIDC discovery "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): @@ -171,7 +171,7 @@ async def test_load_caller_principal_supports_user_id_and_database_email(): assert user_id_binding is not None assert await _load_caller_principal("user-a", user_id_binding) == "user-a" - with patch( + with patch( # test-quality-ok: caller loading is a lazy database boundary without injection "litellm.proxy._experimental.mcp_server.bridge_token_flow.load_active_user_by_id", new=AsyncMock(side_effect=["no_active_key", SimpleNamespace(user_email="alice@example.com")]), ): @@ -183,18 +183,18 @@ async def test_load_caller_principal_supports_user_id_and_database_email(): async def test_load_stored_refresh_token_returns_credential_and_fails_closed(): get_credential: Final = AsyncMock(return_value={"refresh_token": "rt-1"}) with ( - patch( + patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy._experimental.mcp_server.db.get_user_oauth_credential", new=get_credential, ), - patch( + patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy.utils.get_prisma_client_or_throw", return_value="prisma", ), ): assert await _load_stored_refresh_token("user-a", "srv-1") == "rt-1" - with patch( + with patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy.utils.get_prisma_client_or_throw", side_effect=RuntimeError("database unavailable"), ): From eddb29d90ed45849967c2318f15c2edd3e991d8f Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 3 Sep 2026 11:33:44 +0200 Subject: [PATCH 036/411] fix(proxy): dedup latest health checks in SQL and gate the DB save per window --- litellm/constants.py | 1 + litellm/proxy/db/health_check_latest.py | 96 +++++++++ litellm/proxy/proxy_server.py | 58 +++++- litellm/proxy/utils.py | 52 +---- .../proxy/db/test_health_check_latest.py | 111 ++++++++++ .../proxy_server/test_background_health.py | 81 ++++++++ .../proxy/test_health_check_functions.py | 190 ++++++++++-------- .../test_prisma_client_health.py | 74 ++++--- 8 files changed, 502 insertions(+), 161 deletions(-) create mode 100644 litellm/proxy/db/health_check_latest.py create mode 100644 tests/test_litellm/proxy/db/test_health_check_latest.py diff --git a/litellm/constants.py b/litellm/constants.py index ef9329b9dfc..1f5bd673997 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1619,6 +1619,7 @@ CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data" MAVVRIK_FOCUS_EXPORT_JOB_NAME: Final = "mavvrik_focus_export_usage_data" CLOUDZERO_MAX_FETCHED_DATA_RECORDS: Final = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)) SPEND_LOG_CLEANUP_JOB_NAME: Final = "spend_log_cleanup" +BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME: Final = "background_health_check_db_save" KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job" EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME: Final = "litellm_expired_ui_session_key_cleanup_job" WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job" diff --git a/litellm/proxy/db/health_check_latest.py b/litellm/proxy/db/health_check_latest.py new file mode 100644 index 00000000000..35bc838379c --- /dev/null +++ b/litellm/proxy/db/health_check_latest.py @@ -0,0 +1,96 @@ +""" +Latest health-check row per model, deduplicated by Postgres. + +prisma-client-py's ``find_many(distinct=...)`` dedups client-side: the emitted +SQL carries no DISTINCT, so the whole append-only history table streams to the +worker on every call. ``SELECT DISTINCT ON`` keeps the transfer at one row per +(model_id, model_name) and is served by the matching descending index. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Final + +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, field_validator + +from litellm._logging import verbose_proxy_logger + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +LATEST_HEALTH_CHECKS_SQL: Final = """ +SELECT DISTINCT ON ("model_id", "model_name") + "health_check_id", "model_name", "model_id", "status", + "healthy_count", "unhealthy_count", "error_message", + "response_time_ms", "details", "checked_by", + "checked_at", "created_at", "updated_at" +FROM "LiteLLM_HealthCheckTable" +ORDER BY "model_id" ASC, "model_name" ASC, "checked_at" DESC +""" + +LATEST_HEALTH_CHECKS_FOR_MODELS_SQL: Final = """ +SELECT DISTINCT ON ("model_id", "model_name") + "health_check_id", "model_name", "model_id", "status", + "healthy_count", "unhealthy_count", "error_message", + "response_time_ms", "details", "checked_by", + "checked_at", "created_at", "updated_at" +FROM "LiteLLM_HealthCheckTable" +WHERE "model_name" = ANY($1) +ORDER BY "model_id" ASC, "model_name" ASC, "checked_at" DESC +""" + + +class LatestHealthCheckRow(BaseModel): + model_config = ConfigDict(frozen=True, protected_namespaces=()) + + health_check_id: str + model_name: str + model_id: str | None = None + status: str + healthy_count: int = 0 + unhealthy_count: int = 0 + error_message: str | None = None + response_time_ms: float | None = None + details: JsonValue | None = None + checked_by: str | None = None + checked_at: datetime + created_at: datetime + updated_at: datetime + + @field_validator("details", mode="before") + @classmethod + def _decode_json_text(cls, value: object) -> object: + return json.loads(value) if isinstance(value, str) else value + + @field_validator("checked_at", "created_at", "updated_at") + @classmethod + def _assume_utc(cls, value: datetime) -> datetime: + return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value + + +_ROWS_ADAPTER: Final = TypeAdapter(tuple[LatestHealthCheckRow, ...]) + + +async def fetch_latest_health_checks(prisma_client: PrismaClient) -> tuple[LatestHealthCheckRow, ...]: + try: + rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_SQL) + return _ROWS_ADAPTER.validate_python(rows) + except Exception as query_err: # noqa: BLE001 # health decorates other reads; a driver error must not fail them + verbose_proxy_logger.error("Error getting all latest health checks: %s", query_err) + return () + + +async def fetch_latest_health_checks_for_models( + prisma_client: PrismaClient, model_names: Sequence[str] +) -> tuple[LatestHealthCheckRow, ...]: + if not model_names: + return () + try: + rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, list(model_names)) + return _ROWS_ADAPTER.validate_python(rows) + except Exception as query_err: # noqa: BLE001 # a paged model list must not fail on its health decoration + verbose_proxy_logger.error("Error getting latest health checks for models: %s", query_err) + return () diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 27132c90e05..0e330272bdf 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16,7 +16,16 @@ import threading import time import traceback import warnings -from collections.abc import AsyncGenerator, AsyncIterator, Callable, Collection, Mapping, MutableMapping, Sequence +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Awaitable, + Callable, + Collection, + Mapping, + MutableMapping, + Sequence, +) from datetime import datetime, timedelta, timezone from types import MappingProxyType, UnionType from typing import ( @@ -50,6 +59,7 @@ from litellm.constants import ( AIOHTTP_NEEDS_CLEANUP_CLOSED, AIOHTTP_TTL_DNS_CACHE, AUDIO_SPEECH_CHUNK_SIZE, + BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, BASE_MCP_ROUTE, DAILY_TAG_SPEND_BATCH_MULTIPLIER, DEFAULT_MAX_RECURSE_DEPTH, @@ -224,7 +234,7 @@ def generate_feedback_box(): import contextlib from collections import defaultdict from contextlib import asynccontextmanager -from functools import lru_cache +from functools import lru_cache, partial import litellm import litellm._redis @@ -402,6 +412,7 @@ from litellm.proxy.config_resolvers.alerting import ( ) from litellm.proxy.container_endpoints.endpoints import router as container_router from litellm.proxy.credential_endpoints.endpoints import router as credential_router +from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import ( SPEND_LOG_CLEANUP_BOUND_SETTINGS, SpendLogCleanup, @@ -3580,12 +3591,35 @@ async def _run_direct_health_check_with_instrumentation( raise AssertionError("perform_health_check rejected every optional argument") +async def _window_gated_health_check_db_save( + save: Callable[[], Awaitable[None]], + pod_lock_manager: PodLockManager | None, + lock_ttl: int | None, +) -> None: + """ + Persist at most once per window fleet-wide: the lock is the "this window's save is + done" marker, so it is deliberately never released and expires with the interval. + """ + if pod_lock_manager is not None and pod_lock_manager.redis_cache is not None: + acquired: Final = await pod_lock_manager.acquire_lock( + cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, + ttl=lock_ttl, + allow_reentrant=False, + ) + if not acquired: + verbose_proxy_logger.debug("background_health_check_db_save_skipped another pod persisted this window") + return + await save() + + def _schedule_background_health_check_db_save( prisma_client, shared_health_manager, model_list: list, healthy_endpoints: list, unhealthy_endpoints: list, + pod_lock_manager: PodLockManager | None = None, + lock_ttl: int | None = None, ): """Fire-and-forget: persist health check results to DB if prisma is available.""" if prisma_client is None: @@ -3598,16 +3632,16 @@ def _schedule_background_health_check_db_save( checked_by: Final = shared_health_manager.pod_id if shared_health_manager is not None else "background_health_check" start_time: Final = time_module.time() - asyncio.create_task( - _save_background_health_checks_to_db( - prisma_client, - model_list, - healthy_endpoints, - unhealthy_endpoints, - start_time, - checked_by=checked_by, - ) + save: Final = partial( + _save_background_health_checks_to_db, + prisma_client, + model_list, + healthy_endpoints, + unhealthy_endpoints, + start_time, + checked_by=checked_by, ) + asyncio.create_task(_window_gated_health_check_db_save(save, pod_lock_manager, lock_ttl)) def _get_endpoint_exception_status(endpoint: dict, exceptions: dict) -> int: @@ -3914,6 +3948,8 @@ async def _run_background_health_check(): _llm_model_list, healthy_endpoints, unhealthy_endpoints, + pod_lock_manager=proxy_logging_obj.db_spend_update_writer.pod_lock_manager, + lock_ttl=health_check_interval, ) # Write health state to router cache for health-check-driven routing diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cab2bd6d9db..01530762800 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -121,6 +121,11 @@ from litellm.proxy.db.exception_handler import ( PrismaDBExceptionHandler, call_with_db_reconnect_retry, ) +from litellm.proxy.db.health_check_latest import ( + LatestHealthCheckRow, + fetch_latest_health_checks, + fetch_latest_health_checks_for_models, +) from litellm.proxy.db.log_db_metrics import log_db_metrics from litellm.proxy.db.prisma_client import ( PrismaWrapper, @@ -5962,48 +5967,13 @@ class PrismaClient: verbose_proxy_logger.error("Error getting health check history: %s", e) return [] - async def get_all_latest_health_checks(self) -> "Sequence[prisma_models.LiteLLM_HealthCheckTable]": - """ - Get the latest health check for each model. + async def get_all_latest_health_checks(self) -> tuple[LatestHealthCheckRow, ...]: + """Latest health check per (model_id, model_name), deduplicated in Postgres.""" + return await fetch_latest_health_checks(self) - Uses DB-level DISTINCT ON (model_id, model_name) with ORDER BY checked_at DESC - (via Prisma ``distinct`` + ``order``) so we never load the full history into memory. - """ - try: - return await HealthCheckRepository(self).table.find_many( - distinct=["model_id", "model_name"], - order=[ - {"model_id": "asc"}, - {"model_name": "asc"}, - {"checked_at": "desc"}, - ], - ) - except Exception as e: - verbose_proxy_logger.error("Error getting all latest health checks: %s", e) - return [] - - async def get_latest_health_checks_for_models( - self, model_names: "Sequence[str]" - ) -> "Sequence[prisma_models.LiteLLM_HealthCheckTable]": - """ - Get the latest health check for each of the named models. - - Same DISTINCT ON as ``get_all_latest_health_checks``, bounded to the models asked - about, so a paged caller reads health for its page instead of for the whole table. - """ - if not model_names: - return () - latest_first: Final = (("model_id", "asc"), ("model_name", "asc"), ("checked_at", "desc")) - order: Final = [{field: direction} for field, direction in latest_first] # mutable-ok: prisma order is a list - try: - return await HealthCheckRepository(self).table.find_many( - where={"model_name": {"in": list(model_names)}}, # mutable-ok: prisma filters are dicts and lists - distinct=["model_id", "model_name"], # mutable-ok: prisma distinct takes a list - order=order, - ) - except Exception as e: # noqa: BLE001 # health decorates a list; a driver error must not fail the page - verbose_proxy_logger.error("Error getting latest health checks for models: %s", e) - return () + async def get_latest_health_checks_for_models(self, model_names: Sequence[str]) -> tuple[LatestHealthCheckRow, ...]: + """Same as ``get_all_latest_health_checks``, bounded to the named models.""" + return await fetch_latest_health_checks_for_models(self, model_names) ### HELPER FUNCTIONS ### diff --git a/tests/test_litellm/proxy/db/test_health_check_latest.py b/tests/test_litellm/proxy/db/test_health_check_latest.py new file mode 100644 index 00000000000..529e4d8f2e9 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_health_check_latest.py @@ -0,0 +1,111 @@ +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy.db.health_check_latest import ( + LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, + LATEST_HEALTH_CHECKS_SQL, + fetch_latest_health_checks, + fetch_latest_health_checks_for_models, +) + + +def _prisma(rows): + prisma = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=rows) + return prisma + + +def _raw_row(**overrides): + row = { + "health_check_id": "hc-1", + "model_name": "gpt-4", + "model_id": "deployment-abc", + "status": "healthy", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + "response_time_ms": 12.5, + "details": None, + "checked_by": "pod-1", + "checked_at": "2026-08-25T00:00:00+00:00", + "created_at": "2026-08-25T00:00:00+00:00", + "updated_at": "2026-08-25T00:00:00+00:00", + } + return {**row, **overrides} + + +@pytest.mark.asyncio +async def test_fetch_all_runs_one_distinct_on_query_with_no_parameters(): + """The dedup must be in the SQL: prisma find_many(distinct=...) streams the whole history table.""" + prisma = _prisma([]) + assert await fetch_latest_health_checks(prisma) == () + assert prisma.db.query_raw.await_args.args == (LATEST_HEALTH_CHECKS_SQL,) + assert 'DISTINCT ON ("model_id", "model_name")' in LATEST_HEALTH_CHECKS_SQL + assert '"checked_at" DESC' in LATEST_HEALTH_CHECKS_SQL + + +@pytest.mark.asyncio +async def test_raw_datetimes_come_back_tz_aware_with_or_without_an_offset(): + """The save path subtracts checked_at from datetime.now(timezone.utc); a naive value would TypeError.""" + naive = _raw_row(health_check_id="hc-naive", model_id=None, checked_at="2026-08-25T00:00:00") + aware = _raw_row(health_check_id="hc-aware", checked_at="2026-08-25T01:00:00+02:00") + rows = await fetch_latest_health_checks(_prisma([naive, aware])) + assert {row.health_check_id: (row.model_id, row.checked_at) for row in rows} == { + "hc-naive": (None, datetime(2026, 8, 25, 0, 0, tzinfo=timezone.utc)), + "hc-aware": ("deployment-abc", datetime(2026, 8, 24, 23, 0, tzinfo=timezone.utc)), + } + + +@pytest.mark.asyncio +async def test_json_details_decode_from_text_and_pass_through_as_dict(): + rows = await fetch_latest_health_checks( + _prisma( + [ + _raw_row(health_check_id="text", details='{"region": "eu"}'), + _raw_row(health_check_id="dict", details={"region": "us"}), + _raw_row(health_check_id="none", details=None), + ] + ) + ) + assert {row.health_check_id: row.details for row in rows} == { + "text": {"region": "eu"}, + "dict": {"region": "us"}, + "none": None, + } + + +@pytest.mark.asyncio +async def test_fetch_all_degrades_to_no_rows_when_the_query_fails(): + prisma = _prisma([]) + prisma.db.query_raw.side_effect = RuntimeError("db down") + assert await fetch_latest_health_checks(prisma) == () + + +@pytest.mark.asyncio +async def test_fetch_all_degrades_to_no_rows_for_a_malformed_row(): + assert await fetch_latest_health_checks(_prisma([{"unexpected": "shape"}])) == () + + +@pytest.mark.asyncio +async def test_fetch_for_models_binds_the_page_as_the_only_parameter(): + prisma = _prisma([_raw_row()]) + rows = await fetch_latest_health_checks_for_models(prisma, ("gpt-4", "claude-opus")) + assert prisma.db.query_raw.await_args.args == (LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, ["gpt-4", "claude-opus"]) + assert [row.model_name for row in rows] == ["gpt-4"] + assert 'WHERE "model_name" = ANY($1)' in LATEST_HEALTH_CHECKS_FOR_MODELS_SQL + + +@pytest.mark.asyncio +async def test_fetch_for_models_skips_the_database_for_an_empty_page(): + prisma = _prisma([]) + assert await fetch_latest_health_checks_for_models(prisma, ()) == () + prisma.db.query_raw.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_fetch_for_models_degrades_to_no_rows_when_the_query_fails(): + prisma = _prisma([]) + prisma.db.query_raw.side_effect = RuntimeError("db down") + assert await fetch_latest_health_checks_for_models(prisma, ("gpt-4",)) == () diff --git a/tests/test_litellm/proxy/proxy_server/test_background_health.py b/tests/test_litellm/proxy/proxy_server/test_background_health.py index 990844369f7..4874f75752c 100644 --- a/tests/test_litellm/proxy/proxy_server/test_background_health.py +++ b/tests/test_litellm/proxy/proxy_server/test_background_health.py @@ -20,6 +20,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest import litellm.proxy.proxy_server as proxy_server +from litellm.constants import BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME from litellm.proxy.proxy_server import ( _adaptive_router_flusher_loop, _get_endpoint_exception_status, @@ -245,6 +246,86 @@ async def test_schedule_background_health_check_db_save_invalid_no_event_loop_ra ) +def _lock_manager(redis_cache, acquired): + manager = MagicMock() + manager.redis_cache = redis_cache + manager.acquire_lock = AsyncMock(return_value=acquired) + manager.release_lock = AsyncMock() + return manager + + +def _capture_saves(monkeypatch): + saves = [] + + async def _fake_save(*_args, **kwargs): + saves.append(kwargs) + + import litellm.proxy.health_endpoints._health_endpoints as he + + monkeypatch.setattr(he, "_save_background_health_checks_to_db", _fake_save) + return saves + + +def _schedule_with(lock_manager): + _schedule_background_health_check_db_save( + prisma_client=MagicMock(), + shared_health_manager=None, + model_list=[], + healthy_endpoints=[], + unhealthy_endpoints=[], + pod_lock_manager=lock_manager, + lock_ttl=300, + ) + + +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_skips_a_window_another_pod_persisted(monkeypatch): + saves = _capture_saves(monkeypatch) + lock_manager = _lock_manager(redis_cache=MagicMock(), acquired=False) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert saves == [] + + +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_holds_the_window_lock_for_the_whole_interval(monkeypatch): + """The lock is the "saved this window" marker: never reentrant, TTL = interval, and never released.""" + saves = _capture_saves(monkeypatch) + lock_manager = _lock_manager(redis_cache=MagicMock(), acquired=True) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert normalize( + { + "saves": len(saves), + "lock_request": lock_manager.acquire_lock.await_args.kwargs, + "released": lock_manager.release_lock.await_count, + } + ) == { + "saves": 1, + "lock_request": { + "cronjob_id": BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, + "ttl": 300, + "allow_reentrant": False, + }, + "released": 0, + } + + +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_runs_ungated_without_redis(monkeypatch): + saves = _capture_saves(monkeypatch) + lock_manager = _lock_manager(redis_cache=None, acquired=True) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert (len(saves), lock_manager.acquire_lock.await_count) == (1, 0) + + # --------------------------------------------------------------------------- # _get_endpoint_exception_status # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index fdae11d517a..9adaddb032d 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -6,6 +6,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.db.health_check_latest import LatestHealthCheckRow from litellm.proxy.health_endpoints._health_endpoints import ( _aggregate_health_check_results, _build_model_param_to_info_mapping, @@ -13,6 +15,7 @@ from litellm.proxy.health_endpoints._health_endpoints import ( _save_background_health_checks_to_db, _save_health_check_results_if_changed, _save_health_check_to_db, + latest_health_checks_endpoint, ) from litellm.proxy.utils import PrismaClient @@ -451,96 +454,125 @@ async def test_save_background_health_checks_to_db_exception_handling(): # Function should complete without raising -@pytest.mark.asyncio -async def test_get_all_latest_health_checks_with_model_id(mock_prisma): - """Test get_all_latest_health_checks properly groups by model_id""" - mock_check2 = MagicMock() - mock_check2.model_id = "model-456" - mock_check2.model_name = "gpt-3.5-turbo" - mock_check2.checked_at = datetime.now(timezone.utc) - timedelta(minutes=5) - - mock_check3 = MagicMock() - mock_check3.model_id = "model-123" - mock_check3.model_name = "gpt-3.5-turbo" - mock_check3.checked_at = datetime.now(timezone.utc) - timedelta( - minutes=1 - ) # Latest for model-123 - - # Order by checked_at desc - mock_prisma.db.litellm_healthchecktable.find_many = AsyncMock( - return_value=[mock_check3, mock_check2] - ) - - result = await mock_prisma.get_all_latest_health_checks() - - # Should return 2 unique models (by model_id) - assert len(result) == 2 - - # Should have latest check for each model_id - model_ids = {check.model_id for check in result} - assert "model-123" in model_ids - assert "model-456" in model_ids - - # model-123 should have the latest check (1 minute ago) - model123_check = next(c for c in result if c.model_id == "model-123") - assert model123_check.checked_at == mock_check3.checked_at +def _raw_latest_row(model_name: str, model_id, checked_at: datetime) -> dict: + return { + "health_check_id": f"hc-{model_id or 'no-id'}-{model_name}", + "model_name": model_name, + "model_id": model_id, + "status": "healthy", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + "response_time_ms": 10.0, + "details": None, + "checked_by": "pod-1", + "checked_at": checked_at.isoformat(), + "created_at": checked_at.isoformat(), + "updated_at": checked_at.isoformat(), + } @pytest.mark.asyncio -async def test_get_all_latest_health_checks_without_model_id(mock_prisma): - """Test get_all_latest_health_checks groups by model_name when model_id is None""" - mock_check2 = MagicMock() - mock_check2.model_id = None - mock_check2.model_name = "gpt-3.5-turbo" - mock_check2.checked_at = datetime.now(timezone.utc) - timedelta(minutes=1) # Latest - - mock_prisma.db.litellm_healthchecktable.find_many = AsyncMock( - return_value=[mock_check2] - ) - - result = await mock_prisma.get_all_latest_health_checks() - - # Should return 1 unique model (by model_name) - assert len(result) == 1 - assert result[0].model_name == "gpt-3.5-turbo" - assert result[0].checked_at == mock_check2.checked_at # Latest - - -@pytest.mark.asyncio -async def test_get_all_latest_health_checks_same_name_with_and_without_model_id( - mock_prisma, -): +async def test_get_all_latest_health_checks_keeps_every_distinct_group_with_its_own_checked_at(mock_prisma): """ - Same model_name can appear twice after DISTINCT ON: once keyed by (model_id, name) - and once by (NULL, name) — different Postgres groups than a single row with id. + Postgres owns the dedup. (id, name), (other id, name) and (NULL, name) are distinct groups and each row + must arrive typed, with its own checked_at, for the 1h re-save compare and the id-or-name lookup key. """ now = datetime.now(timezone.utc) - with_id = MagicMock() - with_id.model_id = "deployment-abc" - with_id.model_name = "gpt-4" - with_id.checked_at = now - timedelta(minutes=2) - - without_id = MagicMock() - without_id.model_id = None - without_id.model_name = "gpt-4" - without_id.checked_at = now - timedelta(minutes=1) - - mock_prisma.db.litellm_healthchecktable.find_many = AsyncMock( - return_value=[without_id, with_id] + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + _raw_latest_row("gpt-3.5-turbo", "model-123", now - timedelta(minutes=1)), + _raw_latest_row("gpt-3.5-turbo", "model-456", now - timedelta(minutes=5)), + _raw_latest_row("gpt-4", "deployment-abc", now - timedelta(minutes=2)), + _raw_latest_row("gpt-4", None, now - timedelta(minutes=3)), + ] ) result = await mock_prisma.get_all_latest_health_checks() - assert len(result) == 2 - names = {r.model_name for r in result} - assert names == {"gpt-4"} - ids = {r.model_id for r in result} - assert "deployment-abc" in ids - assert None in ids + assert {(check.model_id, check.model_name): check.checked_at for check in result} == { + ("model-123", "gpt-3.5-turbo"): now - timedelta(minutes=1), + ("model-456", "gpt-3.5-turbo"): now - timedelta(minutes=5), + ("deployment-abc", "gpt-4"): now - timedelta(minutes=2), + (None, "gpt-4"): now - timedelta(minutes=3), + } - by_key = {(r.model_id, r.model_name): r for r in result} - assert by_key[("deployment-abc", "gpt-4")].checked_at == with_id.checked_at - assert by_key[(None, "gpt-4")].checked_at == without_id.checked_at + +@pytest.mark.asyncio +async def test_save_background_health_checks_compares_raw_checked_at_against_utc_now(mock_prisma): + """ + Raw rows carry ISO strings and the engine may omit the offset. A naive checked_at would TypeError + inside the 1h compare, be swallowed, and silently stop every save; a stale row must still re-save. + """ + stale = (datetime.now(timezone.utc) - timedelta(hours=2)).replace(tzinfo=None) + fresh = datetime.now(timezone.utc) - timedelta(minutes=5) + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + _raw_latest_row("stale-model", "stale-id", stale), + _raw_latest_row("fresh-model", "fresh-id", fresh), + ] + ) + mock_prisma.save_health_check_result = AsyncMock() + model_list = [ + {"model_name": "stale-model", "model_info": {"id": "stale-id"}, "litellm_params": {"model": "openai/stale"}}, + {"model_name": "fresh-model", "model_info": {"id": "fresh-id"}, "litellm_params": {"model": "openai/fresh"}}, + ] + + await _save_background_health_checks_to_db( + mock_prisma, + model_list, + [{"model": "openai/stale"}, {"model": "openai/fresh"}], + [], + time.time(), + "pod-1", + ) + await asyncio.sleep(0) + + assert [call.kwargs["model_id"] for call in mock_prisma.save_health_check_result.await_args_list] == ["stale-id"] + + +@pytest.mark.asyncio +async def test_latest_health_checks_endpoint_serialises_raw_rows(monkeypatch): + row = LatestHealthCheckRow( + health_check_id="hc-1", + model_name="gpt-4", + model_id="deployment-abc", + status="healthy", + healthy_count=1, + unhealthy_count=0, + error_message=None, + response_time_ms=12.5, + details='{"region": "eu"}', + checked_by="pod-1", + checked_at=datetime(2026, 8, 25), + created_at=datetime(2026, 8, 25, tzinfo=timezone.utc), + updated_at=datetime(2026, 8, 25, tzinfo=timezone.utc), + ) + prisma = MagicMock() + prisma.get_all_latest_health_checks = AsyncMock(return_value=(row,)) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) + + response = await latest_health_checks_endpoint(user_api_key_dict=UserAPIKeyAuth()) + + assert response == { + "latest_health_checks": { + "deployment-abc": { + "health_check_id": "hc-1", + "model_name": "gpt-4", + "model_id": "deployment-abc", + "status": "healthy", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + "response_time_ms": 12.5, + "details": {"region": "eu"}, + "checked_by": "pod-1", + "checked_at": "2026-08-25T00:00:00+00:00", + "created_at": "2026-08-25T00:00:00+00:00", + } + }, + "total_models": 1, + } @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py index 9f48ba68b4f..fbe9934f06c 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py @@ -22,6 +22,10 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from litellm.proxy.db.health_check_latest import ( + LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, + LATEST_HEALTH_CHECKS_SQL, +) from litellm.proxy.utils import PrismaClient @@ -261,36 +265,49 @@ async def test_get_health_check_history_db_error_returns_empty_list( assert await prisma_client.get_health_check_history() == [] +def _raw_health_check_row(model_name: str = "gpt-4", model_id: str | None = "deployment-abc") -> dict[str, Any]: + return { + "health_check_id": f"hc-{model_name}", + "model_name": model_name, + "model_id": model_id, + "status": "healthy", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + "response_time_ms": 12.5, + "details": None, + "checked_by": "pod-1", + "checked_at": "2026-08-25T00:00:00+00:00", + "created_at": "2026-08-25T00:00:00+00:00", + "updated_at": "2026-08-25T00:00:00+00:00", + } + + @pytest.mark.asyncio -async def test_get_all_latest_health_checks_uses_distinct( +async def test_get_all_latest_health_checks_dedups_in_postgres( prisma_client: PrismaClient, ) -> None: - rows = [MagicMock(name=f"row-{i}") for i in range(3)] - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=rows) + """A revert to prisma find_many(distinct=...) streams the whole history table; the SQL must own the DISTINCT.""" + prisma_client.db.query_raw = AsyncMock(return_value=[_raw_health_check_row()]) result = await prisma_client.get_all_latest_health_checks() - kwargs = prisma_client.db.litellm_healthchecktable.find_many.await_args.kwargs actual = { - "len": len(result), - "distinct": kwargs["distinct"], - "order_len": len(kwargs["order"]), - "first_order": kwargs["order"][0], + "query": prisma_client.db.query_raw.await_args.args, + "rows": [(row.model_id, row.model_name, row.status) for row in result], + "row_type": type(result[0]).__name__, } assert actual == { - "len": 3, - "distinct": ["model_id", "model_name"], - "order_len": 3, - "first_order": {"model_id": "asc"}, + "query": (LATEST_HEALTH_CHECKS_SQL,), + "rows": [("deployment-abc", "gpt-4", "healthy")], + "row_type": "LatestHealthCheckRow", } @pytest.mark.asyncio -async def test_get_all_latest_health_checks_db_error_returns_empty_list( +async def test_get_all_latest_health_checks_db_error_returns_no_rows( prisma_client: PrismaClient, ) -> None: - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock( - side_effect=RuntimeError("oops") - ) - assert await prisma_client.get_all_latest_health_checks() == [] + prisma_client.db.query_raw = AsyncMock(side_effect=RuntimeError("oops")) + assert await prisma_client.get_all_latest_health_checks() == () @pytest.mark.asyncio @@ -298,18 +315,15 @@ async def test_get_latest_health_checks_for_models_bounds_the_query_to_those_mod prisma_client: PrismaClient, ) -> None: """A paged caller reads health for its page; an unbounded read is the bug this exists to avoid.""" - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=[]) - await prisma_client.get_latest_health_checks_for_models(["gpt-5", "claude-opus"]) - kwargs = prisma_client.db.litellm_healthchecktable.find_many.await_args.kwargs + prisma_client.db.query_raw = AsyncMock(return_value=[_raw_health_check_row(model_name="gpt-5")]) + result = await prisma_client.get_latest_health_checks_for_models(["gpt-5", "claude-opus"]) actual = { - "where": kwargs["where"], - "distinct": kwargs["distinct"], - "order": kwargs["order"], + "query": prisma_client.db.query_raw.await_args.args, + "rows": [row.model_name for row in result], } assert actual == { - "where": {"model_name": {"in": ["gpt-5", "claude-opus"]}}, - "distinct": ["model_id", "model_name"], - "order": [{"model_id": "asc"}, {"model_name": "asc"}, {"checked_at": "desc"}], + "query": (LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, ["gpt-5", "claude-opus"]), + "rows": ["gpt-5"], } @@ -317,14 +331,14 @@ async def test_get_latest_health_checks_for_models_bounds_the_query_to_those_mod async def test_get_latest_health_checks_for_models_does_not_query_for_an_empty_page( prisma_client: PrismaClient, ) -> None: - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=[]) + prisma_client.db.query_raw = AsyncMock(return_value=[]) assert await prisma_client.get_latest_health_checks_for_models([]) == () - assert prisma_client.db.litellm_healthchecktable.find_many.await_count == 0 + assert prisma_client.db.query_raw.await_count == 0 @pytest.mark.asyncio -async def test_get_latest_health_checks_for_models_db_error_returns_empty_list( +async def test_get_latest_health_checks_for_models_db_error_returns_no_rows( prisma_client: PrismaClient, ) -> None: - prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(side_effect=RuntimeError("oops")) + prisma_client.db.query_raw = AsyncMock(side_effect=RuntimeError("oops")) assert await prisma_client.get_latest_health_checks_for_models(["gpt-5"]) == () From f980b94923e25e5c01607fcdf2088bd0c232fc9e Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 3 Sep 2026 12:36:04 +0200 Subject: [PATCH 037/411] fix(proxy): release the health check save window lock on failure or cancel --- .../health_endpoints/_health_endpoints.py | 105 +++++++----- litellm/proxy/proxy_server.py | 45 +++-- .../proxy_server/test_background_health.py | 109 +++++++----- .../proxy/test_health_check_functions.py | 155 +++++++++++++----- 4 files changed, 278 insertions(+), 136 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 8b57bdca2fe..91825a5606c 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -7,7 +7,7 @@ import secrets import time import traceback from collections.abc import Iterable, Mapping -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Final, Literal, TypedDict, cast import fastapi @@ -41,6 +41,7 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.db.health_check_latest import LatestHealthCheckRow from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers from litellm.proxy.health_check import ( ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, @@ -697,13 +698,42 @@ def _aggregate_health_check_results( return model_results +class _AggregatedHealthResult(TypedDict): + """One entry of ``_aggregate_health_check_results``: a model's counts for this cycle.""" + + model_name: ReadOnly[str] + model_id: ReadOnly[str | None] + healthy_count: ReadOnly[int] + unhealthy_count: ReadOnly[int] + error_message: ReadOnly[str | None] + + +def _new_health_status(result: _AggregatedHealthResult) -> str: + return "healthy" if result["healthy_count"] > 0 else "unhealthy" + + +def _should_persist_health_check_result( + result: _AggregatedHealthResult, latest_checks_map: Mapping[str, LatestHealthCheckRow] +) -> bool: + """ + True when this result has to be written: no previous row, the status changed, or the + previous row is older than one hour (periodic refresh while the status is stable). + """ + lookup_key: Final = result["model_id"] if result["model_id"] else result["model_name"] + last_check: Final = latest_checks_map.get(lookup_key) + if last_check is None or last_check.status != _new_health_status(result): + return True + time_since_last_check: Final = (datetime.now(timezone.utc) - last_check.checked_at).total_seconds() + return time_since_last_check >= 3600 # 1 hour threshold + + async def _save_health_check_results_if_changed( prisma_client, model_results: dict, latest_checks_map: dict, start_time: float, checked_by: str | None = None, -): +) -> bool: """ Save health check results to database, but only if status changed or >1 hour since last save. @@ -714,47 +744,39 @@ async def _save_health_check_results_if_changed( - Status changes: Immediate write (no delay) - Result: ~92% reduction in DB writes for stable systems, while maintaining real-time updates on changes + The writes are awaited rather than detached so the caller learns whether this cycle's + persistence completed. + Args: prisma_client: Database client model_results: Dictionary of aggregated health check results per model latest_checks_map: Dictionary mapping model_id/model_name to latest health check start_time: Start time of health check for calculating response time checked_by: Identifier for who/what performed the check + + Returns: + True when every row that needed writing was written (including when nothing needed + writing); False when any write failed. """ - for result in model_results.values(): - new_status = "healthy" if result["healthy_count"] > 0 else "unhealthy" - - # Check if we should save this result - should_save = True - lookup_key = result["model_id"] if result["model_id"] else result["model_name"] - if lookup_key in latest_checks_map: - last_check = latest_checks_map[lookup_key] - # Only save if status changed or if it's been a while since last check - if last_check.status == new_status: - # Check if last check was recent (within 1 hour) - if last_check.checked_at: - from datetime import datetime, timezone - - time_since_last_check = (datetime.now(timezone.utc) - last_check.checked_at).total_seconds() - # Only skip if status unchanged AND checked recently (within 1 hour) - # This ensures we still get periodic updates even if status is stable - if time_since_last_check < 3600: # 1 hour threshold - should_save = False - - if should_save: - asyncio.create_task( - prisma_client.save_health_check_result( - model_name=result["model_name"], - model_id=result["model_id"], - status=new_status, - healthy_count=result["healthy_count"], - unhealthy_count=result["unhealthy_count"], - error_message=result["error_message"], - response_time_ms=(time.time() - start_time) * 1000, - details=None, - checked_by=checked_by, - ) - ) + to_write: Final = tuple( + result for result in model_results.values() if _should_persist_health_check_result(result, latest_checks_map) + ) + writes: Final = tuple( + prisma_client.save_health_check_result( + model_name=result["model_name"], + model_id=result["model_id"], + status=_new_health_status(result), + healthy_count=result["healthy_count"], + unhealthy_count=result["unhealthy_count"], + error_message=result["error_message"], + response_time_ms=(time.time() - start_time) * 1000, + details=None, + checked_by=checked_by, + ) + for result in to_write + ) + rows: Final = await asyncio.gather(*writes) + return all(row is not None for row in rows) async def _save_background_health_checks_to_db( @@ -764,7 +786,7 @@ async def _save_background_health_checks_to_db( unhealthy_endpoints: list, start_time: float, checked_by: str | None = None, -): +) -> bool: """ Save background health check results to database for each model. @@ -773,9 +795,13 @@ async def _save_background_health_checks_to_db( OPTIMIZATION: Only saves to database if the status has changed from the last saved check. This dramatically reduces database writes when health status remains stable. + + Returns: + True when this cycle's persistence completed; False when it was skipped or any step + failed. Never raises: a database failure must not break the health check loop. """ if prisma_client is None: - return + return False try: # Step 1: Build mapping from model parameter to model info @@ -798,7 +824,7 @@ async def _save_background_health_checks_to_db( latest_checks_map[key] = check # Step 4: Save aggregated results, but only if status changed - await _save_health_check_results_if_changed( + return await _save_health_check_results_if_changed( prisma_client, model_results, latest_checks_map, @@ -808,6 +834,7 @@ async def _save_background_health_checks_to_db( except Exception as db_error: verbose_proxy_logger.warning("Failed to save background health checks to database: %s", db_error) # Continue execution - don't let database save failure break health checks + return False _PROXY_ADMIN_ROLES: Final = frozenset( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0e330272bdf..88ab36b3d1e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -152,6 +152,7 @@ if TYPE_CHECKING: from prisma import models as prisma_models from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager Span = _Span | Any else: @@ -3592,35 +3593,49 @@ async def _run_direct_health_check_with_instrumentation( async def _window_gated_health_check_db_save( - save: Callable[[], Awaitable[None]], + save: Callable[[], Awaitable[bool]], pod_lock_manager: PodLockManager | None, lock_ttl: int | None, ) -> None: """ - Persist at most once per window fleet-wide: the lock is the "this window's save is - done" marker, so it is deliberately never released and expires with the interval. + Persist at most once per window fleet-wide. A completed save keeps the lock as the + "this window's save is done" marker, so it is deliberately never released and expires + with the interval. A save that reports failure or is cancelled releases the lock so + another pod's cycle in the same window can retry, instead of the fleet going a whole + window without a write. """ - if pod_lock_manager is not None and pod_lock_manager.redis_cache is not None: - acquired: Final = await pod_lock_manager.acquire_lock( - cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, - ttl=lock_ttl, - allow_reentrant=False, + if pod_lock_manager is None or pod_lock_manager.redis_cache is None: + await save() + return + acquired: Final = await pod_lock_manager.acquire_lock( + cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME, + ttl=lock_ttl, + allow_reentrant=False, + ) + if not acquired: + verbose_proxy_logger.debug("background_health_check_db_save_skipped another pod persisted this window") + return + try: + persisted: Final = await save() + except BaseException: + await pod_lock_manager.release_lock(cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME) + raise + if not persisted: + verbose_proxy_logger.warning( + "background_health_check_db_save_incomplete released the window lock so another pod can retry" ) - if not acquired: - verbose_proxy_logger.debug("background_health_check_db_save_skipped another pod persisted this window") - return - await save() + await pod_lock_manager.release_lock(cronjob_id=BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME) def _schedule_background_health_check_db_save( - prisma_client, - shared_health_manager, + prisma_client: PrismaClient | None, + shared_health_manager: "SharedHealthCheckManager | None", model_list: list, healthy_endpoints: list, unhealthy_endpoints: list, pod_lock_manager: PodLockManager | None = None, lock_ttl: int | None = None, -): +) -> None: """Fire-and-forget: persist health check results to DB if prisma is available.""" if prisma_client is None: return diff --git a/tests/test_litellm/proxy/proxy_server/test_background_health.py b/tests/test_litellm/proxy/proxy_server/test_background_health.py index 4874f75752c..d99aa637955 100644 --- a/tests/test_litellm/proxy/proxy_server/test_background_health.py +++ b/tests/test_litellm/proxy/proxy_server/test_background_health.py @@ -112,13 +112,11 @@ async def test_run_direct_health_check_with_instrumentation_returns_results( lambda _gs: {}, ) - healthy, unhealthy, exceptions = ( - await _run_direct_health_check_with_instrumentation( - model_list=[{"model_name": "gpt-4"}], - details=False, - max_concurrency=1, - instrumentation_context={"source": "test"}, - ) + healthy, unhealthy, exceptions = await _run_direct_health_check_with_instrumentation( + model_list=[{"model_name": "gpt-4"}], + details=False, + max_concurrency=1, + instrumentation_context={"source": "test"}, ) assert normalize( @@ -254,11 +252,12 @@ def _lock_manager(redis_cache, acquired): return manager -def _capture_saves(monkeypatch): +def _capture_saves(monkeypatch, persisted=True): saves = [] async def _fake_save(*_args, **kwargs): saves.append(kwargs) + return persisted import litellm.proxy.health_endpoints._health_endpoints as he @@ -266,6 +265,15 @@ def _capture_saves(monkeypatch): return saves +def _cancel_during_save(monkeypatch): + async def _fake_save(*_args, **_kwargs): + raise asyncio.CancelledError() + + import litellm.proxy.health_endpoints._health_endpoints as he + + monkeypatch.setattr(he, "_save_background_health_checks_to_db", _fake_save) + + def _schedule_with(lock_manager): _schedule_background_health_check_db_save( prisma_client=MagicMock(), @@ -315,15 +323,56 @@ async def test_schedule_background_health_check_db_save_holds_the_window_lock_fo } +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_releases_the_window_lock_when_the_save_reports_failure( + monkeypatch, +): + """A failed save must not burn the window: release the lock so another pod's cycle can retry.""" + saves = _capture_saves(monkeypatch, persisted=False) + lock_manager = _lock_manager(redis_cache=MagicMock(), acquired=True) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert normalize( + { + "saves": len(saves), + "release_request": lock_manager.release_lock.await_args.kwargs, + "release_count": lock_manager.release_lock.await_count, + } + ) == { + "saves": 1, + "release_request": {"cronjob_id": BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME}, + "release_count": 1, + } + + +@pytest.mark.asyncio +async def test_schedule_background_health_check_db_save_releases_the_window_lock_when_the_save_is_cancelled( + monkeypatch, +): + """A pod shutting down mid-save releases the lock instead of holding it until the TTL.""" + _cancel_during_save(monkeypatch) + lock_manager = _lock_manager(redis_cache=MagicMock(), acquired=True) + + _schedule_with(lock_manager) + await asyncio.sleep(0) + + assert ( + lock_manager.release_lock.await_args.kwargs, + lock_manager.release_lock.await_count, + ) == ({"cronjob_id": BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME}, 1) + + @pytest.mark.asyncio async def test_schedule_background_health_check_db_save_runs_ungated_without_redis(monkeypatch): - saves = _capture_saves(monkeypatch) + saves = _capture_saves(monkeypatch, persisted=False) lock_manager = _lock_manager(redis_cache=None, acquired=True) _schedule_with(lock_manager) await asyncio.sleep(0) - assert (len(saves), lock_manager.acquire_lock.await_count) == (1, 0) + assert (len(saves), lock_manager.acquire_lock.await_count, lock_manager.release_lock.await_count) == (1, 0, 0) # --------------------------------------------------------------------------- @@ -400,13 +449,9 @@ def test_write_health_state_to_router_cache_sets_states(monkeypatch): _write_health_state_to_router_cache(healthy, unhealthy, exceptions) - fake_router.health_state_cache.set_deployment_health_states.assert_called_once_with( - fake_states - ) + fake_router.health_state_cache.set_deployment_health_states.assert_called_once_with(fake_states) - call_args = fake_router.health_state_cache.set_deployment_health_states.call_args[ - 0 - ][0] + call_args = fake_router.health_state_cache.set_deployment_health_states.call_args[0][0] assert normalize( { "states_keys": sorted(call_args.keys()), @@ -448,9 +493,7 @@ def test_write_health_state_to_router_cache_populates_for_listing_filter(monkeyp fake_router.cooldown_time = 30 monkeypatch.setattr(proxy_server, "llm_router", fake_router) - monkeypatch.setattr( - proxy_server, "general_settings", {"model_list_healthy_only": True} - ) + monkeypatch.setattr(proxy_server, "general_settings", {"model_list_healthy_only": True}) fake_states = {"m1": {"is_healthy": True}, "m2": {"is_healthy": False}} @@ -484,9 +527,7 @@ def test_write_health_state_to_router_cache_populates_for_listing_filter(monkeyp {"m2": SimpleNamespace(status_code=500)}, ) - fake_router.health_state_cache.set_deployment_health_states.assert_called_once_with( - fake_states - ) + fake_router.health_state_cache.set_deployment_health_states.assert_called_once_with(fake_states) assert cooldowns == [] assert failures == [] @@ -496,9 +537,7 @@ def test_write_health_state_to_router_cache_swallows_internal_failures(monkeypat fake_router = MagicMock() fake_router.enable_health_check_routing = True fake_router.health_check_ignore_transient_errors = False - fake_router.health_state_cache.set_deployment_health_states.side_effect = ( - RuntimeError("cache exploded") - ) + fake_router.health_state_cache.set_deployment_health_states.side_effect = RuntimeError("cache exploded") monkeypatch.setattr(proxy_server, "llm_router", fake_router) @@ -528,9 +567,7 @@ async def test_adaptive_router_flusher_loop_flushes_each_router(monkeypatch): from litellm.types.router import TaggedPreRoutingStrategy fake_router = MagicMock() - fake_router.adaptive_routers = { - "alpha": [TaggedPreRoutingStrategy(tags=(), strategy=fake_ar)] - } + fake_router.adaptive_routers = {"alpha": [TaggedPreRoutingStrategy(tags=(), strategy=fake_ar)]} monkeypatch.setattr(proxy_server, "llm_router", fake_router) monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) @@ -628,12 +665,8 @@ async def test_run_background_health_check_runs_one_cycle_then_cancels(monkeypat "_run_direct_health_check_with_instrumentation", _fake_direct, ) - monkeypatch.setattr( - proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None - ) - monkeypatch.setattr( - proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None - ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) monkeypatch.setattr( proxy_server, "health_check_filter_kwargs_from_general_settings", @@ -711,12 +744,8 @@ async def test_run_background_health_check_probes_only_listed_model_groups(monke "_run_direct_health_check_with_instrumentation", _fake_direct, ) - monkeypatch.setattr( - proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None - ) - monkeypatch.setattr( - proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None - ) + monkeypatch.setattr(proxy_server, "_schedule_background_health_check_db_save", lambda *a, **kw: None) + monkeypatch.setattr(proxy_server, "_write_health_state_to_router_cache", lambda *a, **kw: None) monkeypatch.setattr( proxy_server, "health_check_filter_kwargs_from_general_settings", diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index 9adaddb032d..5aa80213134 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -24,12 +24,8 @@ from litellm.proxy.utils import PrismaClient def mock_prisma(): """Simplified mock PrismaClient with bound methods""" client = MagicMock() - client.db.litellm_healthchecktable.create = AsyncMock( - return_value={"id": "test-id"} - ) - client.db.litellm_healthchecktable.find_many = AsyncMock( - return_value=[{"id": "1", "model_name": "test"}] - ) + client.db.litellm_healthchecktable.create = AsyncMock(return_value={"id": "test-id"}) + client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=[{"id": "1", "model_name": "test"}]) # Bind actual methods import types @@ -55,14 +51,10 @@ def mock_prisma(): ("healthy", 1, 0, False), # Database error case ], ) -async def test_save_health_check_result( - mock_prisma, status, healthy, unhealthy, should_succeed -): +async def test_save_health_check_result(mock_prisma, status, healthy, unhealthy, should_succeed): """Test health check result saving with various scenarios""" if not should_succeed: - mock_prisma.db.litellm_healthchecktable.create.side_effect = Exception( - "DB Error" - ) + mock_prisma.db.litellm_healthchecktable.create.side_effect = Exception("DB Error") result = await mock_prisma.save_health_check_result( model_name="test-model", @@ -190,9 +182,7 @@ def test_aggregate_health_check_results(): {"model": "gpt-4", "error": "Rate limit exceeded"}, ] - result = _aggregate_health_check_results( - model_param_to_info, healthy_endpoints, unhealthy_endpoints - ) + result = _aggregate_health_check_results(model_param_to_info, healthy_endpoints, unhealthy_endpoints) # Check gpt-3.5-turbo is healthy gpt35_key = ("model-123", "gpt-3.5-turbo") @@ -223,9 +213,7 @@ def test_aggregate_health_check_results_multiple_endpoints(): ] unhealthy_endpoints = [] - result = _aggregate_health_check_results( - model_param_to_info, healthy_endpoints, unhealthy_endpoints - ) + result = _aggregate_health_check_results(model_param_to_info, healthy_endpoints, unhealthy_endpoints) key = ("model-123", "gpt-3.5-turbo") assert result[key]["healthy_count"] == 2 @@ -401,7 +389,7 @@ async def test_save_background_health_checks_to_db(): start_time = 1234567890.0 - await _save_background_health_checks_to_db( + persisted = await _save_background_health_checks_to_db( mock_prisma, model_list, healthy_endpoints, @@ -410,7 +398,8 @@ async def test_save_background_health_checks_to_db(): "background_health_check", ) - # Should call get_all_latest_health_checks and save_health_check_result + # Should call get_all_latest_health_checks and save_health_check_result, and report completion + assert persisted is True mock_prisma.get_all_latest_health_checks.assert_called_once() mock_prisma.save_health_check_result.assert_called_once() @@ -421,22 +410,112 @@ async def test_save_background_health_checks_to_db(): assert call_kwargs["checked_by"] == "background_health_check" +def _two_model_results(): + return { + ("model-1", "gpt-4"): { + "model_name": "gpt-4", + "model_id": "model-1", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + }, + ("model-2", "gpt-4o"): { + "model_name": "gpt-4o", + "model_id": "model-2", + "healthy_count": 0, + "unhealthy_count": 1, + "error_message": "boom", + }, + } + + +@pytest.mark.asyncio +async def test_save_health_check_results_if_changed_awaits_every_write_and_reports_success(): + """Writes are awaited, not detached, so the caller can tell the cycle's persistence completed.""" + mock_prisma = MagicMock() + mock_prisma.save_health_check_result = AsyncMock(return_value={"id": "row"}) + + persisted = await _save_health_check_results_if_changed( + mock_prisma, _two_model_results(), {}, 1234567890.0, "background_health_check" + ) + + assert (persisted, mock_prisma.save_health_check_result.await_count) == (True, 2) + + +@pytest.mark.asyncio +async def test_save_health_check_results_if_changed_reports_failure_when_a_write_returns_none(): + """save_health_check_result swallows DB errors and returns None; that must surface as False.""" + mock_prisma = MagicMock() + mock_prisma.save_health_check_result = AsyncMock(side_effect=[{"id": "row"}, None]) + + persisted = await _save_health_check_results_if_changed( + mock_prisma, _two_model_results(), {}, 1234567890.0, "background_health_check" + ) + + assert (persisted, mock_prisma.save_health_check_result.await_count) == (False, 2) + + +@pytest.mark.asyncio +async def test_save_health_check_results_if_changed_reports_success_when_nothing_needed_writing(): + mock_prisma = MagicMock() + mock_prisma.save_health_check_result = AsyncMock() + model_results = { + ("model-1", "gpt-4"): { + "model_name": "gpt-4", + "model_id": "model-1", + "healthy_count": 1, + "unhealthy_count": 0, + "error_message": None, + }, + } + latest_checks_map = { + "model-1": MagicMock(status="healthy", checked_at=datetime.now(timezone.utc) - timedelta(minutes=5)), + } + + persisted = await _save_health_check_results_if_changed( + mock_prisma, model_results, latest_checks_map, 1234567890.0, "background_health_check" + ) + + assert (persisted, mock_prisma.save_health_check_result.await_count) == (True, 0) + + +def _one_model_setup(): + model_list = [ + { + "model_name": "gpt-3.5-turbo", + "model_info": {"id": "model-123"}, + "litellm_params": {"model": "gpt-3.5-turbo"}, + }, + ] + return model_list, [{"model": "gpt-3.5-turbo"}], [] + + +@pytest.mark.asyncio +async def test_save_background_health_checks_to_db_returns_false_when_a_write_fails(): + mock_prisma = MagicMock() + mock_prisma.get_all_latest_health_checks = AsyncMock(return_value=[]) + mock_prisma.save_health_check_result = AsyncMock(return_value=None) + model_list, healthy_endpoints, unhealthy_endpoints = _one_model_setup() + + persisted = await _save_background_health_checks_to_db( + mock_prisma, model_list, healthy_endpoints, unhealthy_endpoints, 1234567890.0, "background_health_check" + ) + + assert (persisted, mock_prisma.save_health_check_result.await_count) == (False, 1) + + @pytest.mark.asyncio async def test_save_background_health_checks_to_db_no_prisma(): """Test graceful handling when no prisma client""" - result = await _save_background_health_checks_to_db( - None, [], [], [], 0.0, "background_health_check" - ) - assert result is None + result = await _save_background_health_checks_to_db(None, [], [], [], 0.0, "background_health_check") + assert result is False @pytest.mark.asyncio async def test_save_background_health_checks_to_db_exception_handling(): """Test exception handling in background health check save""" mock_prisma = MagicMock() - mock_prisma.get_all_latest_health_checks = AsyncMock( - side_effect=Exception("DB Error") - ) + mock_prisma.get_all_latest_health_checks = AsyncMock(side_effect=Exception("DB Error")) model_list = [ { @@ -446,12 +525,13 @@ async def test_save_background_health_checks_to_db_exception_handling(): }, ] - # Should not raise exception, should handle gracefully - await _save_background_health_checks_to_db( + # Must not raise (the health check loop has to survive a DB outage) but must report + # the failure, so the window lock can be released for another pod to retry + persisted = await _save_background_health_checks_to_db( mock_prisma, model_list, [], [], 0.0, "background_health_check" ) - # Function should complete without raising + assert persisted is False def _raw_latest_row(model_name: str, model_id, checked_at: datetime) -> dict: @@ -660,12 +740,7 @@ def test_parse_background_health_check_model_groups_unset_returns_none(): assert parse_background_health_check_model_groups(None) is None assert parse_background_health_check_model_groups({}) is None - assert ( - parse_background_health_check_model_groups( - {"background_health_check_model_groups": None} - ) - is None - ) + assert parse_background_health_check_model_groups({"background_health_check_model_groups": None}) is None def test_parse_background_health_check_model_groups_list_returns_frozenset(): @@ -682,9 +757,7 @@ def test_parse_background_health_check_model_groups_malformed_raises(bad_value): from litellm.proxy.health_check import parse_background_health_check_model_groups with pytest.raises(ValueError, match="must be a list of model group names"): - parse_background_health_check_model_groups( - {"background_health_check_model_groups": bad_value} - ) + parse_background_health_check_model_groups({"background_health_check_model_groups": bad_value}) def test_filter_deployments_to_model_groups(): @@ -697,9 +770,7 @@ def test_filter_deployments_to_model_groups(): ] assert filter_deployments_to_model_groups(model_list, None) == tuple(model_list) - assert filter_deployments_to_model_groups( - model_list, frozenset({"prod-openai"}) - ) == (model_list[0], model_list[2]) + assert filter_deployments_to_model_groups(model_list, frozenset({"prod-openai"})) == (model_list[0], model_list[2]) assert filter_deployments_to_model_groups(model_list, frozenset()) == () From 512e730f2fbbb4a832a42ca65b8b29c41c77508d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:30:48 -0700 Subject: [PATCH 038/411] fix(azure_ai): add passthrough config so router-model relays reach the deployment's own endpoint Every /azure_ai// relay failed with HTTP 500 because azure_ai had no passthrough config. The new AzureAIPassthroughConfig strips the router-model prefix from the relayed path, forwards to the deployment's api_base with its own credential (api-key on Foundry and Azure OpenAI hosts, Bearer elsewhere, Entra as the fallback), and delegates chat/completions cost logging to the Azure passthrough config. The router's provider inference now receives the deployment's api_base so an OpenAI-family model on a Foundry resource stays azure_ai instead of flipping to azure through the AZURE_AI_API_BASE env var. --- basedpyright-code-budget.json | 6 +- litellm/llms/azure_ai/chat/transformation.py | 12 +- litellm/llms/azure_ai/common_utils.py | 7 + .../azure_ai/passthrough/transformation.py | 106 +++++++++++ .../base_llm/passthrough/transformation.py | 3 +- litellm/router.py | 2 + litellm/utils.py | 6 + ruff-strict-budget.json | 2 +- ...est_azure_ai_passthrough_transformation.py | 175 ++++++++++++++++++ .../passthrough/test_passthrough_main.py | 115 ++++++++++++ type-discipline-budget.json | 2 +- 11 files changed, 423 insertions(+), 13 deletions(-) create mode 100644 litellm/llms/azure_ai/passthrough/transformation.py create mode 100644 tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 669107bb5b1..f31d620557d 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15284 + "limit": 15283 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,10 +105,10 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38309 + "limit": 38308 }, "reportUnknownParameterType": { - "limit": 19621 + "limit": 19620 }, "reportUnknownVariableType": { "limit": 29844 diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index f2d405e9a17..4c0a2e1f546 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -2,7 +2,6 @@ import copy import enum import re from typing import TYPE_CHECKING, Final, cast -from urllib.parse import urlparse import httpx from httpx import Response @@ -15,7 +14,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( filter_value_from_dict, ) from litellm.llms.azure.common_utils import BaseAzureLLM -from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base +from litellm.llms.azure_ai.common_utils import ( + api_key_header_for_base, + is_foundry_model_inference_base, +) from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error from litellm.llms.openai.openai import OpenAIConfig @@ -99,11 +101,7 @@ class AzureAIStudioConfig(OpenAIConfig): """ Returns True if the request should use `api-key` header for authentication. """ - parsed_url: Final = urlparse(api_base) - host: Final = parsed_url.hostname - if host and (host.endswith(".services.ai.azure.com") or host.endswith(".openai.azure.com")): - return True - return False + return api_key_header_for_base(api_base) == "api-key" def get_complete_url( self, diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py index aa34bab5b2e..0665b3f64c5 100644 --- a/litellm/llms/azure_ai/common_utils.py +++ b/litellm/llms/azure_ai/common_utils.py @@ -19,6 +19,13 @@ def is_foundry_model_inference_base(api_base: str) -> bool: return "/openai/deployments" not in parsed.path +def api_key_header_for_base(api_base: str | None) -> AzureAIApiKeyHeader: + host: Final = urlparse(api_base).hostname if api_base else None + if host and (host.endswith(".services.ai.azure.com") or host.endswith(".openai.azure.com")): + return "api-key" + return "Authorization" + + def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None: """ Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment. diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py new file mode 100644 index 00000000000..afe0b814524 --- /dev/null +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm.llms.azure_ai.common_utils import ( + AzureFoundryModelInfo, + api_key_header_for_base, + get_azure_ai_auth_headers, +) +from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig +from litellm.types.llms.openai import AllMessageValues + +if TYPE_CHECKING: + from httpx import URL, Response + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import CostResponseTypes + + +def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> str: + path: Final = endpoint.lstrip("/") + for model_name in model_names: + if not model_name: + continue + if path == model_name: + return "" + if path.startswith(f"{model_name}/"): + return path[len(model_name) + 1 :] + return path + + +class PassthroughMetadata(BaseModel): + model_config = ConfigDict(extra="ignore") + + model_group: str = "" + + +def model_group_from(litellm_params: Mapping[str, object]) -> str: + try: + return PassthroughMetadata.model_validate(litellm_params.get("litellm_metadata")).model_group + except ValidationError: + return "" + + +class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): + def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool: + return request_data.get("stream") is True + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + endpoint: str, + request_query_params: Mapping[str, object] | None, + litellm_params: Mapping[str, object], + ) -> tuple[URL, str]: + base_target_url: Final = self.get_api_base(api_base) + if base_target_url is None: + raise ValueError("Azure AI api base not found: set `api_base` on the deployment or AZURE_AI_API_BASE") + + native_endpoint: Final = strip_leading_model_segment(endpoint, (model, model_group_from(litellm_params))) + return ( + self.format_url(native_endpoint, base_target_url, request_query_params), + base_target_url, + ) + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: base class contract returns dict for httpx + auth_headers: Final = get_azure_ai_auth_headers( + api_key=api_key, + litellm_params=litellm_params, + api_key_header=api_key_header_for_base(api_base), + ) + return {**headers, **auth_headers} # mutable-ok: base class contract returns dict for httpx + + def logging_non_streaming_response( + self, + model: str, + custom_llm_provider: str, + httpx_response: Response, + request_data: Mapping[str, object], + logging_obj: Logging, + endpoint: str, + ) -> CostResponseTypes | None: + from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig + + return AzurePassthroughConfig().logging_non_streaming_response( # pyright: ignore[reportUnknownMemberType] # the Azure config still types request_data as a bare dict + model=model, + custom_llm_provider=custom_llm_provider, + httpx_response=httpx_response, + request_data=dict(request_data), # mutable-ok: AzurePassthroughConfig wants a dict + logging_obj=logging_obj, + endpoint=endpoint, + ) diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index adbf2e126fb..34533e4c603 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -1,4 +1,5 @@ from abc import abstractmethod +from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Optional, Union from ..base_utils import BaseLLMModelInfo @@ -23,7 +24,7 @@ class BasePassthroughConfig(BaseLLMModelInfo): self, endpoint: str, base_target_url: str, - request_query_params: dict | None, + request_query_params: Mapping[str, object] | None, ) -> "URL": """ Helper function to add query params to the url diff --git a/litellm/router.py b/litellm/router.py index 6c7611c6236..b84f790287f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5051,6 +5051,7 @@ class Router: _, inferred_custom_llm_provider, _, _ = get_llm_provider( model=data["model"], custom_llm_provider=custom_llm_provider, + api_base=data.get("api_base"), ) custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider except Exception: @@ -5570,6 +5571,7 @@ class Router: _, inferred_custom_llm_provider, _, _ = get_llm_provider( model=data["model"], custom_llm_provider=custom_llm_provider, + api_base=data.get("api_base"), ) custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider except Exception: diff --git a/litellm/utils.py b/litellm/utils.py index 9d20d32d147..0f055308e3d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8831,6 +8831,12 @@ class ProviderConfigManager: ) return AzurePassthroughConfig() + elif LlmProviders.AZURE_AI == provider: + from litellm.llms.azure_ai.passthrough.transformation import ( + AzureAIPassthroughConfig, + ) + + return AzureAIPassthroughConfig() elif LlmProviders.GIGACHAT == provider: from litellm.llms.gigachat.passthrough.transformation import ( GigaChatPassthroughConfig, diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 70408ea022b..a011230c2d3 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -201,7 +201,7 @@ "limit": 310 }, "SIM103": { - "limit": 119 + "limit": 118 }, "SIM113": { "limit": 3 diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py new file mode 100644 index 00000000000..e1abbe1a7e1 --- /dev/null +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -0,0 +1,175 @@ +import json +from unittest.mock import MagicMock + +import httpx +import pytest + +import litellm +from litellm.llms.azure_ai.passthrough.transformation import AzureAIPassthroughConfig +from litellm.types.utils import LlmProviders, ModelResponse +from litellm.utils import ProviderConfigManager + +FOUNDRY_BASE = "https://my-resource.services.ai.azure.com" + + +@pytest.fixture(autouse=True) +def clear_azure_ai_env(monkeypatch): + for env_var in ("AZURE_AI_API_BASE", "AZURE_AI_API_KEY", "AZURE_AD_TOKEN", "AZURE_API_KEY"): + monkeypatch.delenv(env_var, raising=False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "api_key", None) + + +def test_provider_config_manager_resolves_azure_ai_passthrough_config(): + config = ProviderConfigManager.get_provider_passthrough_config(model="Cohere-parse-v5", provider=LlmProviders.AZURE_AI) + + assert isinstance(config, AzureAIPassthroughConfig) + + +def test_router_model_prefix_is_stripped_and_native_path_kept_verbatim(): + url, base = AzureAIPassthroughConfig().get_complete_url( + api_base=FOUNDRY_BASE, + api_key=None, + model="Cohere-parse-v5", + endpoint="Cohere-parse-v5/providers/cohere/v2/parse", + request_query_params=None, + litellm_params={}, + ) + + assert str(url) == f"{FOUNDRY_BASE}/providers/cohere/v2/parse" + assert base == FOUNDRY_BASE + + +def test_model_group_prefix_is_stripped_when_router_metadata_names_it(): + url, _ = AzureAIPassthroughConfig().get_complete_url( + api_base=FOUNDRY_BASE, + api_key=None, + model="Cohere-parse-v5", + endpoint="/parse-alias/providers/cohere/v2/parse", + request_query_params=None, + litellm_params={"litellm_metadata": {"model_group": "parse-alias"}}, + ) + + assert str(url) == f"{FOUNDRY_BASE}/providers/cohere/v2/parse" + + +def test_model_inside_the_path_stays_and_query_params_are_forwarded(): + url, _ = AzureAIPassthroughConfig().get_complete_url( + api_base=f"{FOUNDRY_BASE}/", + api_key=None, + model="gpt-5.4-mini", + endpoint="openai/deployments/gpt-5.4-mini/chat/completions", + request_query_params={"api-version": "2024-10-21"}, + litellm_params={}, + ) + + assert str(url) == f"{FOUNDRY_BASE}/openai/deployments/gpt-5.4-mini/chat/completions?api-version=2024-10-21" + + +def test_missing_api_base_raises_instead_of_building_a_relative_url(): + with pytest.raises(ValueError, match="AZURE_AI_API_BASE"): + AzureAIPassthroughConfig().get_complete_url( + api_base=None, + api_key=None, + model="Cohere-parse-v5", + endpoint="Cohere-parse-v5/providers/cohere/v2/parse", + request_query_params=None, + litellm_params={}, + ) + + +def _auth_headers(api_key: str | None, api_base: str, litellm_params: dict | None = None) -> dict: + return AzureAIPassthroughConfig().validate_environment( + headers={"content-type": "application/json"}, + model="Cohere-parse-v5", + messages=[], + optional_params={}, + litellm_params=litellm_params or {}, + api_key=api_key, + api_base=api_base, + ) + + +def test_foundry_host_gets_the_api_key_header(): + headers = _auth_headers(api_key="deployment-key", api_base=FOUNDRY_BASE) + + assert headers == {"content-type": "application/json", "api-key": "deployment-key"} + + +def test_serverless_host_gets_a_bearer_token(): + headers = _auth_headers(api_key="deployment-key", api_base="https://cohere-parse.eastus.models.ai.azure.com") + + assert headers["Authorization"] == "Bearer deployment-key" + assert "api-key" not in headers + + +def test_entra_token_is_used_when_the_deployment_has_no_api_key(): + headers = _auth_headers(api_key=None, api_base=FOUNDRY_BASE, litellm_params={"azure_ad_token": "entra-token"}) + + assert headers["Authorization"] == "Bearer entra-token" + + +def test_no_credentials_at_all_raises(): + with pytest.raises(ValueError, match="Missing Azure AI credentials"): + _auth_headers(api_key=None, api_base=FOUNDRY_BASE) + + +@pytest.mark.parametrize( + "request_data, expected", + [({"stream": True}, True), ({"stream": False}, False), ({}, False)], +) +def test_is_streaming_request_reads_the_stream_flag(request_data, expected): + assert AzureAIPassthroughConfig().is_streaming_request(endpoint="models/chat/completions", request_data=request_data) is expected + + +def _chat_completion_response() -> httpx.Response: + body = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.4-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}, + } + return httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=json.dumps(body).encode("utf-8"), + request=httpx.Request("POST", f"{FOUNDRY_BASE}/models/chat/completions"), + ) + + +def test_chat_completions_relay_yields_a_model_response_for_cost_tracking(): + result = AzureAIPassthroughConfig().logging_non_streaming_response( + model="gpt-5.4-mini", + custom_llm_provider="azure_ai", + httpx_response=_chat_completion_response(), + request_data={"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "hi"}]}, + logging_obj=MagicMock(), + endpoint="models/chat/completions", + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "hi" + assert result.usage.prompt_tokens == 10 + assert result.usage.completion_tokens == 8 + + +def test_non_chat_relay_yields_no_cost_response(): + parse_response = httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=b'{"id":"parse-1","pages":[]}', + request=httpx.Request("POST", f"{FOUNDRY_BASE}/providers/cohere/v2/parse"), + ) + + result = AzureAIPassthroughConfig().logging_non_streaming_response( + model="Cohere-parse-v5", + custom_llm_provider="azure_ai", + httpx_response=parse_response, + request_data={"model": "Cohere-parse-v5"}, + logging_obj=MagicMock(), + endpoint="providers/cohere/v2/parse", + ) + + assert result is None diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index 1950c37a12e..546cff18b5d 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -873,3 +873,118 @@ def test_llm_passthrough_route_propagates_allm_passthrough_route_to_logging_obj( assert captured_litellm_params.get("allm_passthrough_route") is True assert LitellmLogging._is_sync_litellm_request(captured_litellm_params) is False + + +FOUNDRY_BASE = "https://my-resource.services.ai.azure.com" + + +def _foundry_parse_response() -> httpx.Response: + return httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=b'{"id":"parse-1","pages":[]}', + request=httpx.Request("POST", f"{FOUNDRY_BASE}/providers/cohere/v2/parse"), + ) + + +def test_azure_ai_relay_reaches_the_deployment_with_its_own_credential(): + """ + Regression for LIT-7022: azure_ai had no passthrough config, so every + /azure_ai// relay raised "Provider azure_ai not found" + before a request was built. + """ + client = HTTPHandler() + + with patch.object(client.client, "send", return_value=_foundry_parse_response()) as mock_send: + response = llm_passthrough_route( + model="azure_ai/Cohere-parse-v5", + endpoint="Cohere-parse-v5/providers/cohere/v2/parse", + method="POST", + custom_llm_provider="azure_ai", + api_base=FOUNDRY_BASE, + api_key="deployment-key", + json={"model": "Cohere-parse-v5", "document": {"type": "image_url", "image_url": "https://x/y.png"}}, + client=client, + litellm_logging_obj=MagicMock(), + ) + + sent = mock_send.call_args.kwargs["request"] + assert str(sent.url) == f"{FOUNDRY_BASE}/providers/cohere/v2/parse" + assert sent.headers["api-key"] == "deployment-key" + assert json.loads(sent.content)["model"] == "Cohere-parse-v5" + assert response.status_code == 200 + + +@pytest.mark.asyncio +async def test_router_relays_azure_ai_model_through_the_deployment_api_base(): + router = litellm.Router( + model_list=[ + { + "model_name": "foundry-parse", + "litellm_params": { + "model": "azure_ai/Cohere-parse-v5", + "api_base": FOUNDRY_BASE, + "api_key": "deployment-key", + }, + } + ] + ) + async_client = AsyncHTTPHandler() + + with patch.object(async_client.client, "send", AsyncMock(return_value=_foundry_parse_response())) as mock_send: + response = await router.allm_passthrough_route( + model="foundry-parse", + method="POST", + endpoint="foundry-parse/providers/cohere/v2/parse", + json={"model": "foundry-parse", "document": {"type": "image_url", "image_url": "https://x/y.png"}}, + client=async_client, + ) + + sent = mock_send.call_args.kwargs["request"] + assert str(sent.url) == f"{FOUNDRY_BASE}/providers/cohere/v2/parse" + assert sent.headers["api-key"] == "deployment-key" + assert json.loads(sent.content)["model"] == "Cohere-parse-v5" + assert response.status_code == 200 + + +@pytest.mark.asyncio +async def test_router_relays_an_openai_model_on_a_foundry_base_as_azure_ai(monkeypatch): + monkeypatch.setenv("AZURE_AI_API_BASE", "https://unrelated.openai.azure.com") + router = litellm.Router( + model_list=[ + { + "model_name": "foundry-gpt", + "litellm_params": { + "model": "azure_ai/gpt-5.4-mini", + "api_base": FOUNDRY_BASE, + "api_key": "deployment-key", + }, + } + ] + ) + async_client = AsyncHTTPHandler() + upstream = httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=( + b'{"id":"chatcmpl-1","object":"chat.completion","model":"gpt-5.4-mini",' + b'"choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"hi"}}],' + b'"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}' + ), + request=httpx.Request("POST", f"{FOUNDRY_BASE}/models/chat/completions"), + ) + + with patch.object(async_client.client, "send", AsyncMock(return_value=upstream)) as mock_send: + await router.allm_passthrough_route( + model="foundry-gpt", + method="POST", + endpoint="foundry-gpt/models/chat/completions", + request_query_params={"api-version": "2024-05-01-preview"}, + json={"model": "foundry-gpt", "messages": [{"role": "user", "content": "hi"}]}, + client=async_client, + ) + + sent = mock_send.call_args.kwargs["request"] + assert str(sent.url) == f"{FOUNDRY_BASE}/models/chat/completions?api-version=2024-05-01-preview" + assert sent.headers["api-key"] == "deployment-key" + assert json.loads(sent.content)["model"] == "gpt-5.4-mini" diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 3d01c08e8eb..7d12721175f 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22326 + "limit": 22325 }, "LIT002": { "limit": 26748 From a276690ce200bebe3547c110ac9a830b32052173 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:32:26 -0700 Subject: [PATCH 039/411] fix(router): keep a model's own provider prefix for generic SDK calls Generic passthrough calls inferred the provider from the bare model name, so an azure_ai/gpt-* deployment on an Azure OpenAI host flipped to azure and get_llm_provider re-prefixed the deployment name into azure_ai/gpt-5.4-mini, a 404 DeploymentNotFound. provider_for_generic_call takes the declared custom_llm_provider first, then the model's own prefix, and only infers for unprefixed models --- litellm/router.py | 24 ++--------- litellm/router_utils/common_utils.py | 27 ++++++++++++ .../test_router_utils_common_utils.py | 18 ++++++++ tests/test_litellm/test_router.py | 43 +++++++++++++++++++ 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index b84f790287f..9b7a7ee7ce8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -135,6 +135,7 @@ from litellm.router_utils.common_utils import ( _is_proxy_admin_request, filter_team_based_models, filter_web_search_deployments, + provider_for_generic_call, resolve_model_group_alias, truncate_fallback_error_detail, warn_on_provider_credential_mismatch, @@ -5045,17 +5046,7 @@ class Router: kwargs=kwargs, model=model, model_name=model_name ) - # Get custom_llm_provider from deployment params - try: - custom_llm_provider = data.get("custom_llm_provider") - _, inferred_custom_llm_provider, _, _ = get_llm_provider( - model=data["model"], - custom_llm_provider=custom_llm_provider, - api_base=data.get("api_base"), - ) - custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider - except Exception: - custom_llm_provider = None + custom_llm_provider: Final = provider_for_generic_call(data) response_kwargs: Final = { **data, @@ -5566,16 +5557,7 @@ class Router: # Perform pre-call checks for routing strategy self.routing_strategy_pre_call_checks(deployment=deployment) - try: - custom_llm_provider = data.get("custom_llm_provider") - _, inferred_custom_llm_provider, _, _ = get_llm_provider( - model=data["model"], - custom_llm_provider=custom_llm_provider, - api_base=data.get("api_base"), - ) - custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider - except Exception: - custom_llm_provider = None + custom_llm_provider: Final = provider_for_generic_call(data) response: Final = original_function( **{ diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 280a7defcf8..ec3ffa1247e 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Final if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIFileObject +import litellm from litellm._logging import verbose_logger, verbose_router_logger from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError @@ -244,6 +245,32 @@ PROVIDER_SCOPED_CREDENTIAL_PARAMS: Final[Mapping[str, frozenset[str]]] = Mapping ) +def provider_for_generic_call(litellm_params: Mapping[str, object]) -> str | None: + """ + The provider the router hands a deployment's generic SDK call, or None when it cannot be resolved. + + A model that carries its own provider prefix keeps that prefix even where get_llm_provider + would resolve it to a sibling provider (azure_ai/ on an Azure OpenAI host + resolves to azure): the SDK call still receives the prefixed model, and an explicit provider + that contradicts the prefix makes get_llm_provider re-prefix it into a deployment name that + does not exist upstream. + """ + declared: Final = litellm_params.get("custom_llm_provider") + if isinstance(declared, str) and declared: + return declared + model: Final = litellm_params.get("model") + if not isinstance(model, str) or not model: + return None + prefix: Final = model.split("/", 1)[0] + if "/" in model and prefix in litellm.provider_list: + return prefix + try: + _, inferred, _, _ = get_llm_provider(model=model) + except BadRequestError: + return None + return inferred + + def warn_on_provider_credential_mismatch(model_name: str, litellm_params: Mapping[str, object]) -> str | None: """ Warn when a deployment carries one provider's credentials but resolves to another. diff --git a/tests/test_litellm/router_utils/test_router_utils_common_utils.py b/tests/test_litellm/router_utils/test_router_utils_common_utils.py index 30f658d7ea2..ac18b4889dd 100644 --- a/tests/test_litellm/router_utils/test_router_utils_common_utils.py +++ b/tests/test_litellm/router_utils/test_router_utils_common_utils.py @@ -12,6 +12,7 @@ from litellm.router_utils.common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + provider_for_generic_call, resolve_model_group_alias, truncate_fallback_error_detail, PROVIDER_SCOPED_CREDENTIAL_PARAMS, @@ -756,3 +757,20 @@ class TestWarnOnProviderCredentialMismatch: ) is None ) + + +@pytest.mark.parametrize( + ("litellm_params", "expected"), + [ + ({"model": "azure_ai/gpt-5.4-mini", "custom_llm_provider": "azure"}, "azure"), + ({"model": "azure_ai/gpt-5.4-mini", "api_base": "https://my-resource.openai.azure.com"}, "azure_ai"), + ({"model": "cohere/command-r"}, "cohere"), + ({"model": "gpt-5.4-mini"}, "openai"), + ({"model": "no-provider-knows-this-model"}, None), + ({"api_base": "https://my-resource.openai.azure.com"}, None), + ], + ids=["declared_wins", "prefix_beats_host_flip", "prefix_beats_cohere_chat_flip", "unprefixed_inferred", "unknown", "no_model"], +) +def test_provider_for_generic_call(litellm_params, expected, monkeypatch): + monkeypatch.setenv("AZURE_AI_API_BASE", "https://unrelated.openai.azure.com") + assert provider_for_generic_call(litellm_params) == expected diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 31eb46f1458..b220b23c338 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -12938,3 +12938,46 @@ async def test_router_retry_policy_controls_upstream_attempt_count( await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) assert upstream.call_count == expected_upstream_calls + + +@pytest.mark.asyncio +async def test_generic_call_keeps_the_deployment_name_of_an_azure_ai_model_on_an_azure_openai_host(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + { + "model_name": "aoai-gpt", + "litellm_params": { + "model": "azure_ai/gpt-5.4-mini", + "api_base": "https://my-resource.openai.azure.com", + "api_key": "deployment-key", + }, + } + ] + ) + + with respx.mock(assert_all_called=True) as respx_mock: + upstream = respx_mock.post(host="my-resource.openai.azure.com", path__regex=r"^/openai/.*responses$").mock( + return_value=httpx.Response( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.4-mini", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + }, + ) + ) + await router.aresponses(model="aoai-gpt", input="hi") + + assert json.loads(upstream.calls.last.request.content)["model"] == "gpt-5.4-mini" From cd25eb918995ef115aa67e8fd39337e4786bdcd2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:32:27 -0700 Subject: [PATCH 040/411] fix(azure_ai): cost streaming relays and return upstream errors from router relays Streaming chat relays on Azure and azure_ai deployments rebuild the response from the SSE chunks through the OpenAI passthrough assembler, so the spend log carries usage. The router relays keep the JSON body when the Content-Type carries a charset, return the upstream status and body instead of a 500 when the deployment rejects the call, and fall back to the caller's api-version when the deployment sets none. Lint budgets ratcheted to the measured totals --- basedpyright-code-budget.json | 8 +- .../llms/azure/passthrough/transformation.py | 24 ++++- .../azure_ai/passthrough/transformation.py | 18 ++++ .../proxy/common_utils/http_parsing_utils.py | 4 +- .../llm_passthrough_endpoints.py | 53 ++++++----- .../openai_passthrough_logging_handler.py | 3 +- ruff-strict-budget.json | 4 +- .../test_azure_passthrough_transformation.py | 69 +++++++++++++++ ...est_azure_ai_passthrough_transformation.py | 21 +++++ .../test_llm_pass_through_endpoints.py | 87 +++++++++++++++++++ type-discipline-budget.json | 4 +- 11 files changed, 260 insertions(+), 35 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index b446c79f49b..ef1bd675c5e 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5570 }, "reportMissingTypeArgument": { - "limit": 15279 + "limit": 15278 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38281 + "limit": 38276 }, "reportUnknownParameterType": { - "limit": 19582 + "limit": 19581 }, "reportUnknownVariableType": { - "limit": 29829 + "limit": 29825 }, "reportUnnecessaryCast": { "limit": 110 diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 898852e645f..7654958fc0b 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from typing import TYPE_CHECKING, Final, Optional import httpx @@ -43,7 +44,7 @@ class AzurePassthroughConfig(BasePassthroughConfig): api_base=base_target_url, litellm_params=litellm_params, route=endpoint, - default_api_version=litellm_params.get("api_version"), + default_api_version=request_query_params.get("api-version") if request_query_params else None, ) return ( httpx.URL(complete_url), @@ -116,3 +117,24 @@ class AzurePassthroughConfig(BasePassthroughConfig): ) return litellm_model_response + + def handle_logging_collected_chunks( + self, + all_chunks: Sequence[str], + litellm_logging_obj: Logging, + model: str, + custom_llm_provider: str, + endpoint: str, + ) -> Optional["CostResponseTypes"]: + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import ( + OpenAIPassthroughLoggingHandler, + ) + + if "chat/completions" not in endpoint: + return None + + return OpenAIPassthroughLoggingHandler()._build_complete_streaming_response( # pyright: ignore[reportPrivateUsage] # the only OpenAI SSE-to-ModelResponse assembler; reimplementing it would fork the parser + all_chunks=all_chunks, + litellm_logging_obj=litellm_logging_obj, + model=model, + ) diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index afe0b814524..5948c865bad 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -104,3 +104,21 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): logging_obj=logging_obj, endpoint=endpoint, ) + + def handle_logging_collected_chunks( + self, + all_chunks: Sequence[str], + litellm_logging_obj: Logging, + model: str, + custom_llm_provider: str, + endpoint: str, + ) -> CostResponseTypes | None: + from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig + + return AzurePassthroughConfig().handle_logging_collected_chunks( + all_chunks=all_chunks, + litellm_logging_obj=litellm_logging_obj, + model=model, + custom_llm_provider=custom_llm_provider, + endpoint=endpoint, + ) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 54a0f18fd63..635cbf3ba9b 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -39,7 +39,7 @@ def _is_form_content_type(content_type: str) -> bool: return _normalize_media_type(content_type) in _FORM_CONTENT_TYPES -def _is_json_content_type(content_type: str) -> bool: +def is_json_content_type(content_type: str) -> bool: """True iff the body should be parsed as JSON.""" return _normalize_media_type(content_type) == "application/json" @@ -406,7 +406,7 @@ async def get_request_body(request: Request) -> dict[str, Any]: """ if request.method == "POST": content_type: Final = request.headers.get("content-type", "") - if _is_json_content_type(content_type): + if is_json_content_type(content_type): return await _read_request_body(request) elif _is_form_content_type(content_type): return await get_form_data(request) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 6b1d6405a6a..cf7b7fdf07a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -49,6 +49,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_set_request_parsed_body, get_form_data, get_request_body, + is_json_content_type, ) from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, @@ -408,7 +409,7 @@ async def vllm_proxy_route( content=None, data=None, files=None, - json=(request_body if request.headers.get("content-type") == "application/json" else None), + json=(request_body if is_json_content_type(request.headers.get("content-type", "")) else None), params=None, headers=None, cookies=None, @@ -1492,6 +1493,14 @@ async def _relay_upstream_bytes(upstream: AsyncGenerator[bytes, bytes]) -> Async await upstream.aclose() +async def _relay_upstream_response(upstream: httpx.Response) -> Response: + return Response( + content=await upstream.aread(), + status_code=upstream.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers(headers=upstream.headers, custom_headers=None), + ) + + async def _relay_azure_router_model( llm_router: litellm.Router, model: str, @@ -1501,30 +1510,28 @@ async def _relay_azure_router_model( is_streaming_request: bool, user_api_key_dict: UserAPIKeyAuth, ) -> Response: - result: Final = await llm_router.allm_passthrough_route( - model=model, - method=request.method, - endpoint=endpoint, - request_query_params=request.query_params, - request_headers=_safe_get_request_headers(request), - stream=is_streaming_request, - content=None, - data=None, - files=None, - json=(request_body if request.headers.get("content-type") == "application/json" else None), - params=None, - headers=None, - cookies=None, - litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict), - ) + try: + result: Final = await llm_router.allm_passthrough_route( + model=model, + method=request.method, + endpoint=endpoint, + request_query_params=request.query_params, + request_headers=_safe_get_request_headers(request), + stream=is_streaming_request, + content=None, + data=None, + files=None, + json=(request_body if is_json_content_type(request.headers.get("content-type", "")) else None), + params=None, + headers=None, + cookies=None, + litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict), + ) + except httpx.HTTPStatusError as upstream_error: + return await _relay_upstream_response(upstream_error.response) if not is_streaming_request: - upstream: Final = cast(httpx.Response, result) - return Response( - content=await upstream.aread(), - status_code=upstream.status_code, - headers=HttpPassThroughEndpointHelpers.get_response_headers(headers=upstream.headers, custom_headers=None), - ) + return await _relay_upstream_response(cast(httpx.Response, result)) if inspect.isasyncgen(result): sse_headers: Final = {"content-type": "text/event-stream"} diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 5f6489a69ca..a39ea342f0a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -4,6 +4,7 @@ OpenAI Passthrough Logging Handler Handles cost tracking and logging for OpenAI passthrough endpoints, specifically /chat/completions. """ +from collections.abc import Sequence from datetime import datetime from typing import Final from urllib.parse import urlparse @@ -512,7 +513,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): def _build_complete_streaming_response( self, - all_chunks: list[str], + all_chunks: Sequence[str], litellm_logging_obj: LiteLLMLoggingObj, model: str, ) -> ModelResponse | TextCompletionResponse | None: diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 694ce6e86e0..3354610bfcf 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -57,7 +57,7 @@ "limit": 3 }, "BLE001": { - "limit": 2916 + "limit": 2914 }, "C401": { "limit": 8 @@ -201,7 +201,7 @@ "limit": 310 }, "SIM103": { - "limit": 116 + "limit": 115 }, "SIM113": { "limit": 3 diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 29b74c2ee4a..68fbc99b5d5 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -92,3 +92,72 @@ def test_azure_passthrough_logging_non_streaming_response_unknown_endpoint_retur ) assert result is None + + +def _sse_line(payload: dict) -> str: + return "data: " + json.dumps(payload) + + +def _azure_chat_completion_chunks() -> list[str]: + head = {"id": "chatcmpl-abc123", "object": "chat.completion.chunk", "created": 1700000000, "model": "gpt-4.1-mini"} + return [ + _sse_line({**head, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello!"}, "finish_reason": None}]}), + _sse_line({**head, "choices": [{"index": 0, "delta": {"content": " How can I assist?"}, "finish_reason": None}]}), + _sse_line({**head, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}), + _sse_line({**head, "choices": [], "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}}), + "data: [DONE]", + ] + + +def test_azure_passthrough_streaming_chat_chunks_build_the_complete_response(): + response = AzurePassthroughConfig().handle_logging_collected_chunks( + all_chunks=_azure_chat_completion_chunks(), + litellm_logging_obj=MagicMock(), + model="gpt-4.1-mini", + custom_llm_provider="azure", + endpoint="openai/deployments/gpt-4.1-mini/chat/completions", + ) + + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "Hello! How can I assist?" + assert response.usage.prompt_tokens == 10 + assert response.usage.completion_tokens == 8 + + +def test_azure_passthrough_streaming_chunks_for_unknown_endpoint_return_none(): + response = AzurePassthroughConfig().handle_logging_collected_chunks( + all_chunks=_azure_chat_completion_chunks(), + litellm_logging_obj=MagicMock(), + model="gpt-4.1-mini", + custom_llm_provider="azure", + endpoint="openai/deployments/gpt-4.1-mini/embeddings", + ) + + assert response is None + + +def _complete_url(request_query_params: dict, litellm_params: dict) -> httpx.URL: + url, _ = AzurePassthroughConfig().get_complete_url( + api_base="https://my-resource.openai.azure.com", + api_key="key", + model="gpt-4.1-mini", + endpoint="openai/deployments/gpt-4.1-mini/chat/completions", + request_query_params=request_query_params, + litellm_params=litellm_params, + ) + return url + + +def test_azure_passthrough_url_falls_back_to_the_callers_api_version(): + url = _complete_url(request_query_params={"api-version": "2025-04-01-preview"}, litellm_params={}) + + assert url.path == "/openai/deployments/gpt-4.1-mini/chat/completions" + assert url.params["api-version"] == "2025-04-01-preview" + + +def test_azure_passthrough_url_prefers_the_deployments_api_version(): + url = _complete_url( + request_query_params={"api-version": "2025-04-01-preview"}, litellm_params={"api_version": "2024-10-21"} + ) + + assert url.params["api-version"] == "2024-10-21" diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index e1abbe1a7e1..67851d2d58e 100644 --- a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -173,3 +173,24 @@ def test_non_chat_relay_yields_no_cost_response(): ) assert result is None + + +def test_streaming_chat_completion_chunks_are_costed_like_azure(): + head = {"id": "chatcmpl-1", "object": "chat.completion.chunk", "created": 1, "model": "gpt-5.4-mini"} + chunks = [ + "data: " + json.dumps({**head, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}]}), + "data: " + json.dumps({**head, "choices": [], "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}}), + "data: [DONE]", + ] + + response = AzureAIPassthroughConfig().handle_logging_collected_chunks( + all_chunks=chunks, + litellm_logging_obj=MagicMock(), + model="gpt-5.4-mini", + custom_llm_provider="azure_ai", + endpoint="chat/completions", + ) + + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "hi" + assert response.usage.total_tokens == 4 diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index acb45038df0..a2043808b6b 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -5206,3 +5206,90 @@ class TestAzureRouterModelStreamingKeepalive: assert result.headers["x-upstream"] == "kept" assert chunks == [b"data: hello\n\n"] + + +class TestRouterModelRelayUpstreamContract: + def _request(self, content_type: str) -> MagicMock: + request = MagicMock(spec=Request) + request.method = "POST" + request.headers = {"content-type": content_type} + request.query_params = {} + return request + + def _install_router(self, monkeypatch, router, body: dict) -> None: + import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep + import litellm.proxy.proxy_server as proxy_server + + async def fake_get_request_body(_request): + return body + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) + monkeypatch.setattr(ep, "is_passthrough_request_using_router_model", lambda *a, **k: True) + + def _recording_router(self, captured: list[dict]): + class RecordingRouter: + async def allm_passthrough_route(self, **kwargs): + captured.append(kwargs) + return httpx.Response(200, json={"ok": True}) + + return RecordingRouter() + + @pytest.mark.asyncio + async def test_azure_relay_keeps_the_json_body_when_the_content_type_carries_a_charset(self, monkeypatch): + body = {"model": "gpt-5", "messages": [{"role": "user", "content": "hi"}]} + captured: list[dict] = [] + self._install_router(monkeypatch, self._recording_router(captured), body) + + await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=self._request("application/json; charset=utf-8"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert captured[0]["json"] == body + + @pytest.mark.asyncio + async def test_vllm_relay_keeps_the_json_body_when_the_content_type_carries_a_charset(self, monkeypatch): + body = {"model": "router-model", "messages": [{"role": "user", "content": "hi"}]} + captured: list[dict] = [] + self._install_router(monkeypatch, self._recording_router(captured), body) + + await vllm_proxy_route( + endpoint="/chat/completions", + request=self._request("application/json; charset=utf-8"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert captured[0]["json"] == body + + @pytest.mark.asyncio + async def test_azure_relay_returns_the_upstream_status_and_body_when_the_deployment_rejects_the_call( + self, monkeypatch + ): + upstream_body = {"error": {"code": "DeploymentNotFound", "message": "The API deployment does not exist."}} + + class RejectingRouter: + async def allm_passthrough_route(self, **kwargs): + upstream_request = httpx.Request( + "POST", "https://my-azure.openai.azure.com/openai/deployments/gpt-5/chat/completions" + ) + upstream = httpx.Response( + 404, json=upstream_body, headers={"x-ms-request-id": "req-1"}, request=upstream_request + ) + raise httpx.HTTPStatusError("404", request=upstream_request, response=upstream) + + self._install_router(monkeypatch, RejectingRouter(), {"model": "gpt-5", "stream": False}) + + result = await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=self._request("application/json"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert result.status_code == 404 + assert json.loads(result.body) == upstream_body + assert result.headers["x-ms-request-id"] == "req-1" diff --git a/type-discipline-budget.json b/type-discipline-budget.json index bfa055b43e5..30bd580c4e2 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22180 + "limit": 22178 }, "LIT002": { "limit": 26745 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16464 + "limit": 16458 }, "LIT011": { "limit": 5506 From 9ea9aa2e7b1ef97f80e2141f0bc299498e351143 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:00:23 -0700 Subject: [PATCH 041/411] fix(azure_ai): count prompt tokens for streaming relays that carry no usage chunk --- basedpyright-code-budget.json | 8 ++--- .../streaming_chunk_builder_utils.py | 4 +-- .../llms/azure/passthrough/transformation.py | 20 ++++++++++- litellm/main.py | 4 +-- .../openai_passthrough_logging_handler.py | 7 ++-- ruff-strict-budget.json | 4 +-- .../test_azure_passthrough_transformation.py | 33 +++++++++++++++++-- type-discipline-budget.json | 4 +-- 8 files changed, 67 insertions(+), 17 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index ef1bd675c5e..4dc7a0d8671 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5570 }, "reportMissingTypeArgument": { - "limit": 15278 + "limit": 15277 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38276 + "limit": 38271 }, "reportUnknownParameterType": { - "limit": 19581 + "limit": 19580 }, "reportUnknownVariableType": { - "limit": 29825 + "limit": 29821 }, "reportUnnecessaryCast": { "limit": 110 diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 0e01577b20e..1beb06f52e7 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -209,7 +209,7 @@ def apply_grounding_request_counts( class ChunkProcessor: - def __init__(self, chunks: list, messages: list | None = None): + def __init__(self, chunks: list, messages: Sequence | None = None): self.chunks = self._sort_chunks(chunks) self.messages = messages self.first_chunk = chunks[0] @@ -992,7 +992,7 @@ class ChunkProcessor: chunks: Sequence["_UsageBearingChunk | ModelResponse"], model: str, completion_output: str, - messages: list | None = None, + messages: Sequence | None = None, reasoning_tokens: int | None = None, ) -> Usage: """ diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 7654958fc0b..0b8b216faae 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -1,8 +1,9 @@ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final, Optional import httpx from httpx import Response +from pydantic import BaseModel, ValidationError from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure.common_utils import BaseAzureLLM @@ -17,6 +18,22 @@ if TYPE_CHECKING: from litellm.types.utils import CostResponseTypes +class RelayedChatRequest(BaseModel): + messages: Sequence[Mapping[str, object]] | None = None + + +class RelayedCallDetails(BaseModel): + request_data: RelayedChatRequest | None = None + + +def _relayed_messages(litellm_logging_obj: Logging) -> Sequence[Mapping[str, object]] | None: + try: + details: Final = RelayedCallDetails.model_validate(litellm_logging_obj.model_call_details) + except ValidationError: + return None + return details.request_data.messages if details.request_data else None + + class AzurePassthroughConfig(BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: return "stream" in request_data @@ -137,4 +154,5 @@ class AzurePassthroughConfig(BasePassthroughConfig): all_chunks=all_chunks, litellm_logging_obj=litellm_logging_obj, model=model, + messages=_relayed_messages(litellm_logging_obj), ) diff --git a/litellm/main.py b/litellm/main.py index 2929790f2bd..a7fa5560d30 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8553,7 +8553,7 @@ def config_completion(**kwargs): ) -def stream_chunk_builder_text_completion(chunks: list, messages: list | None = None) -> TextCompletionResponse: +def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None = None) -> TextCompletionResponse: id: Final = chunks[0]["id"] object: Final = chunks[0]["object"] created: Final = chunks[0]["created"] @@ -8670,7 +8670,7 @@ def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_o def stream_chunk_builder( chunks: list, - messages: list | None = None, + messages: Sequence | None = None, start_time=None, end_time=None, logging_obj: Optional["Logging"] = None, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index a39ea342f0a..9805c7d7fae 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -4,7 +4,7 @@ OpenAI Passthrough Logging Handler Handles cost tracking and logging for OpenAI passthrough endpoints, specifically /chat/completions. """ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime from typing import Final from urllib.parse import urlparse @@ -516,6 +516,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): all_chunks: Sequence[str], litellm_logging_obj: LiteLLMLoggingObj, model: str, + messages: Sequence[Mapping[str, object]] | None = None, ) -> ModelResponse | TextCompletionResponse | None: """ Builds complete response from raw chunks for OpenAI streaming responses. @@ -559,7 +560,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return None # Build complete response from chunks - complete_streaming_response: Final = litellm.stream_chunk_builder(chunks=all_openai_chunks) + complete_streaming_response: Final = litellm.stream_chunk_builder( + chunks=all_openai_chunks, messages=messages + ) return complete_streaming_response diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 3354610bfcf..cfe737867f1 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -57,7 +57,7 @@ "limit": 3 }, "BLE001": { - "limit": 2914 + "limit": 2912 }, "C401": { "limit": 8 @@ -201,7 +201,7 @@ "limit": 310 }, "SIM103": { - "limit": 115 + "limit": 114 }, "SIM113": { "limit": 3 diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 68fbc99b5d5..d0657679983 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -3,6 +3,8 @@ from unittest.mock import MagicMock import httpx +import litellm + from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig from litellm.types.utils import ModelResponse @@ -101,8 +103,15 @@ def _sse_line(payload: dict) -> str: def _azure_chat_completion_chunks() -> list[str]: head = {"id": "chatcmpl-abc123", "object": "chat.completion.chunk", "created": 1700000000, "model": "gpt-4.1-mini"} return [ - _sse_line({**head, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello!"}, "finish_reason": None}]}), - _sse_line({**head, "choices": [{"index": 0, "delta": {"content": " How can I assist?"}, "finish_reason": None}]}), + _sse_line( + { + **head, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello!"}, "finish_reason": None}], + } + ), + _sse_line( + {**head, "choices": [{"index": 0, "delta": {"content": " How can I assist?"}, "finish_reason": None}]} + ), _sse_line({**head, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}), _sse_line({**head, "choices": [], "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}}), "data: [DONE]", @@ -124,6 +133,26 @@ def test_azure_passthrough_streaming_chat_chunks_build_the_complete_response(): assert response.usage.completion_tokens == 8 +def test_azure_passthrough_streaming_chunks_without_usage_count_prompt_tokens_from_the_relayed_request(): + messages = [{"role": "user", "content": "Say hi in three words"}] + logging_obj = MagicMock() + logging_obj.model_call_details = {"request_data": {"messages": messages, "stream": True}} + + response = AzurePassthroughConfig().handle_logging_collected_chunks( + all_chunks=[chunk for chunk in _azure_chat_completion_chunks() if '"usage"' not in chunk], + litellm_logging_obj=logging_obj, + model="gpt-4.1-mini", + custom_llm_provider="azure", + endpoint="openai/deployments/gpt-4.1-mini/chat/completions", + ) + + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "Hello! How can I assist?" + assert response.usage.prompt_tokens > 0 + assert response.usage.prompt_tokens == litellm.token_counter(model="gpt-4.1-mini", messages=messages) + assert response.usage.completion_tokens > 0 + + def test_azure_passthrough_streaming_chunks_for_unknown_endpoint_return_none(): response = AzurePassthroughConfig().handle_logging_collected_chunks( all_chunks=_azure_chat_completion_chunks(), diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 30bd580c4e2..74b54148418 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22178 + "limit": 22172 }, "LIT002": { "limit": 26745 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16458 + "limit": 16452 }, "LIT011": { "limit": 5506 From bbbdccb82d88f3489a1a311109f86d87bb1782e8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:30:13 -0700 Subject: [PATCH 042/411] fix(azure_ai): count relayed image prompt tokens without fetching the image --- basedpyright-code-budget.json | 8 +++--- .../streaming_chunk_builder_utils.py | 5 +++- litellm/main.py | 3 ++ .../openai_passthrough_logging_handler.py | 2 +- ruff-strict-budget.json | 4 +-- .../test_azure_passthrough_transformation.py | 28 +++++++++++++++++++ type-discipline-budget.json | 4 +-- 7 files changed, 44 insertions(+), 10 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 4dc7a0d8671..f63fb2a497a 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5570 }, "reportMissingTypeArgument": { - "limit": 15277 + "limit": 15276 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38271 + "limit": 38266 }, "reportUnknownParameterType": { - "limit": 19580 + "limit": 19579 }, "reportUnknownVariableType": { - "limit": 29821 + "limit": 29817 }, "reportUnnecessaryCast": { "limit": 110 diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 1beb06f52e7..1dca64f1a80 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -994,6 +994,7 @@ class ChunkProcessor: completion_output: str, messages: Sequence | None = None, reasoning_tokens: int | None = None, + use_default_image_token_count: bool = False, ) -> Usage: """ Calculate usage for the given chunks. @@ -1018,7 +1019,9 @@ class ChunkProcessor: cost: Final[float | None] = calculated_usage_per_chunk["cost"] try: - returned_usage.prompt_tokens = prompt_tokens or token_counter(model=model, messages=messages) + returned_usage.prompt_tokens = prompt_tokens or token_counter( + model=model, messages=messages, use_default_image_token_count=use_default_image_token_count + ) except Exception: # don't allow this failing to block a complete streaming response from being returned print_verbose("token_counter failed, assuming prompt tokens is 0") returned_usage.prompt_tokens = 0 diff --git a/litellm/main.py b/litellm/main.py index a7fa5560d30..d125f1c1d7f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8674,6 +8674,7 @@ def stream_chunk_builder( start_time=None, end_time=None, logging_obj: Optional["Logging"] = None, + use_default_image_token_count: bool = False, ) -> ModelResponse | TextCompletionResponse | None: try: if chunks is None: @@ -8747,6 +8748,7 @@ def stream_chunk_builder( completion_output=completion_output, messages=messages, reasoning_tokens=0, + use_default_image_token_count=use_default_image_token_count, ) setattr(response, "usage", usage) @@ -8924,6 +8926,7 @@ def stream_chunk_builder( completion_output=completion_output, messages=messages, reasoning_tokens=reasoning_tokens, + use_default_image_token_count=use_default_image_token_count, ) setattr(response, "usage", usage) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 9805c7d7fae..e21105f760c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -561,7 +561,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): # Build complete response from chunks complete_streaming_response: Final = litellm.stream_chunk_builder( - chunks=all_openai_chunks, messages=messages + chunks=all_openai_chunks, messages=messages, use_default_image_token_count=True ) return complete_streaming_response diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index cfe737867f1..907317e81b1 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -57,7 +57,7 @@ "limit": 3 }, "BLE001": { - "limit": 2912 + "limit": 2910 }, "C401": { "limit": 8 @@ -201,7 +201,7 @@ "limit": 310 }, "SIM103": { - "limit": 114 + "limit": 113 }, "SIM113": { "limit": 3 diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index d0657679983..b07b7915277 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -153,6 +153,34 @@ def test_azure_passthrough_streaming_chunks_without_usage_count_prompt_tokens_fr assert response.usage.completion_tokens > 0 +def test_azure_passthrough_streaming_chunks_count_remote_image_prompt_tokens_without_fetching_the_image(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this"}, + {"type": "image_url", "image_url": {"url": "http://127.0.0.1:9/doc.png"}}, + ], + } + ] + logging_obj = MagicMock() + logging_obj.model_call_details = {"request_data": {"messages": messages, "stream": True}} + + response = AzurePassthroughConfig().handle_logging_collected_chunks( + all_chunks=[chunk for chunk in _azure_chat_completion_chunks() if '"usage"' not in chunk], + litellm_logging_obj=logging_obj, + model="gpt-4.1-mini", + custom_llm_provider="azure", + endpoint="openai/deployments/gpt-4.1-mini/chat/completions", + ) + + assert isinstance(response, ModelResponse) + assert response.usage.prompt_tokens > 0 + assert response.usage.prompt_tokens == litellm.token_counter( + model="gpt-4.1-mini", messages=messages, use_default_image_token_count=True + ) + + def test_azure_passthrough_streaming_chunks_for_unknown_endpoint_return_none(): response = AzurePassthroughConfig().handle_logging_collected_chunks( all_chunks=_azure_chat_completion_chunks(), diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 74b54148418..78cd84f5b35 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22172 + "limit": 22166 }, "LIT002": { "limit": 26745 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16452 + "limit": 16446 }, "LIT011": { "limit": 5506 From 3f695846b3cbb688d0d581bc64c4da5258546240 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 02:09:25 -0700 Subject: [PATCH 043/411] fix(router): rewrite the passthrough model group as a whole path segment --- .../base_llm/passthrough/transformation.py | 20 ++++++++++++++-- litellm/router.py | 3 ++- tests/test_litellm/test_router.py | 23 +++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index 34533e4c603..d97ecddc75d 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -8,11 +8,27 @@ if TYPE_CHECKING: from httpx import URL, Headers, Response from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.types.utils import CostResponseTypes + from litellm.types.utils import CostResponseTypes, StandardPassThroughResponseObject from ..chat.transformation import BaseLLMException +def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> str: + path: Final = endpoint.lstrip("/") + for model_name in model_names: + if not model_name: + continue + if path == model_name: + return "" + if path.startswith(f"{model_name}/"): + return path[len(model_name) + 1 :] + return path + + +def replace_path_segment(endpoint: str, segment: str, replacement: str) -> str: + return "/".join(replacement if part == segment else part for part in endpoint.split("/")) + + class BasePassthroughConfig(BaseLLMModelInfo): @abstractmethod def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: @@ -104,7 +120,7 @@ class BasePassthroughConfig(BaseLLMModelInfo): request_data: dict, logging_obj: "LiteLLMLoggingObj", endpoint: str, - ) -> Optional["CostResponseTypes"]: + ) -> Optional["CostResponseTypes | StandardPassThroughResponseObject"]: pass def handle_logging_collected_chunks( diff --git a/litellm/router.py b/litellm/router.py index 9b7a7ee7ce8..273c1f9d99c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -86,6 +86,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( mask_credentials_in_payload, mask_sensitive_structure, ) +from litellm.llms.base_llm.passthrough.transformation import replace_path_segment from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, vector_store_request_metadata, @@ -5010,7 +5011,7 @@ class Router: # If get_llm_provider fails, fall back to using model_name as-is replacement_model_name = model_name - kwargs["endpoint"] = kwargs["endpoint"].replace(model, replacement_model_name) + kwargs["endpoint"] = replace_path_segment(kwargs["endpoint"], model, replacement_model_name) return kwargs async def _ageneric_api_call_with_fallbacks_helper(self, model: str, original_generic_function: Callable, **kwargs): diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b220b23c338..0f4e199694d 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -4469,6 +4469,29 @@ def test_get_deployment_model_info_base_model_merge_priority(): print("✓ Base model merge priority test passed!") +def test_add_deployment_model_to_endpoint_rewrites_whole_path_segments_only(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt", + "litellm_params": { + "model": "azure_ai/gpt-5.4-mini", + "api_base": "https://my-resource.services.ai.azure.com", + "api_key": "key", + }, + } + ], + ) + + result = router._add_deployment_model_to_endpoint_for_llm_passthrough_route( + kwargs={"endpoint": "gpt/openai/deployments/gpt-4o/chat/completions", "custom_llm_provider": "azure_ai"}, + model="gpt", + model_name="azure_ai/gpt-5.4-mini", + ) + + assert result["endpoint"] == "gpt-5.4-mini/openai/deployments/gpt-4o/chat/completions" + + def test_add_deployment_model_to_endpoint_for_llm_passthrough_route(): """ Test that _add_deployment_model_to_endpoint_for_llm_passthrough_route correctly strips bedrock provider prefix From df6fb9e5d95e97b7b207cd278d107e82b134fe45 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 02:09:26 -0700 Subject: [PATCH 044/411] fix(azure_ai): relay from the Foundry root and log non-chat relays for spend tracking --- .../llms/azure/passthrough/transformation.py | 14 +-- .../azure_ai/passthrough/transformation.py | 69 +++++++++++---- ...est_azure_ai_passthrough_transformation.py | 87 +++++++++++++++++-- 3 files changed, 140 insertions(+), 30 deletions(-) diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 0b8b216faae..983033a8b14 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -7,7 +7,11 @@ from pydantic import BaseModel, ValidationError from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure.common_utils import BaseAzureLLM -from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig +from litellm.llms.base_llm.passthrough.transformation import ( + BasePassthroughConfig, + replace_path_segment, + strip_leading_model_segment, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams @@ -36,7 +40,7 @@ def _relayed_messages(litellm_logging_obj: Logging) -> Sequence[Mapping[str, obj class AzurePassthroughConfig(BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: - return "stream" in request_data + return bool(request_data.get("stream")) def get_complete_url( self, @@ -54,13 +58,13 @@ class AzurePassthroughConfig(BasePassthroughConfig): litellm_metadata: Final = litellm_params.get("litellm_metadata") or {} model_group: Final = litellm_metadata.get("model_group") - if model_group and model_group in endpoint: - endpoint = endpoint.replace(model_group, model) + routed_endpoint: Final = replace_path_segment(endpoint, model_group, model) if model_group else endpoint + native_endpoint: Final = strip_leading_model_segment(routed_endpoint, (model,)) complete_url: Final = BaseAzureLLM._get_base_azure_url( api_base=base_target_url, litellm_params=litellm_params, - route=endpoint, + route=native_endpoint, default_api_version=request_query_params.get("api-version") if request_query_params else None, ) return ( diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index 5948c865bad..4776bbe0755 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -1,17 +1,20 @@ from __future__ import annotations from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Final -from pydantic import BaseModel, ConfigDict, ValidationError +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.llms.azure_ai.common_utils import ( AzureFoundryModelInfo, api_key_header_for_base, get_azure_ai_auth_headers, ) -from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig +from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, strip_leading_model_segment from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import StandardPassThroughResponseObject if TYPE_CHECKING: from httpx import URL, Response @@ -20,16 +23,7 @@ if TYPE_CHECKING: from litellm.types.utils import CostResponseTypes -def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> str: - path: Final = endpoint.lstrip("/") - for model_name in model_names: - if not model_name: - continue - if path == model_name: - return "" - if path.startswith(f"{model_name}/"): - return path[len(model_name) + 1 :] - return path +EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({}) class PassthroughMetadata(BaseModel): @@ -45,9 +39,44 @@ def model_group_from(litellm_params: Mapping[str, object]) -> str: return "" +def api_version_from(litellm_params: Mapping[str, object]) -> str | None: + try: + return TypeAdapter(str | None).validate_python(litellm_params.get("api_version")) + except ValidationError: + return None + + +def foundry_root(api_base: str) -> str: + url: Final = httpx.URL(api_base) + segments: Final = tuple(segment for segment in url.path.split("/") if segment) + root_segments: Final = segments[: segments.index("models")] if "models" in segments else segments + return str(url.copy_with(path="/" + "/".join(root_segments), query=None)).rstrip("/") + + +def relay_query_params( + request_query_params: Mapping[str, object] | None, + deployment_api_version: str | None, + api_base: str, +) -> Mapping[str, object] | None: + if request_query_params and "api-version" in request_query_params: + return request_query_params + api_version: Final = deployment_api_version or httpx.URL(api_base).params.get("api-version") + if api_version is None: + return request_query_params + return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version}) + + +def relayed_body(httpx_response: Response) -> str | dict: + try: + body: Final[object] = httpx_response.json() + except ValueError: + return httpx_response.text + return body if isinstance(body, dict) else httpx_response.text + + class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool: - return request_data.get("stream") is True + return bool(request_data.get("stream")) def get_complete_url( self, @@ -62,11 +91,12 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): if base_target_url is None: raise ValueError("Azure AI api base not found: set `api_base` on the deployment or AZURE_AI_API_BASE") + root: Final = foundry_root(base_target_url) native_endpoint: Final = strip_leading_model_segment(endpoint, (model, model_group_from(litellm_params))) - return ( - self.format_url(native_endpoint, base_target_url, request_query_params), - base_target_url, + query_params: Final = relay_query_params( + request_query_params, api_version_from(litellm_params), base_target_url ) + return (self.format_url(native_endpoint, root, query_params), root) def validate_environment( self, @@ -93,10 +123,10 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): request_data: Mapping[str, object], logging_obj: Logging, endpoint: str, - ) -> CostResponseTypes | None: + ) -> CostResponseTypes | StandardPassThroughResponseObject | None: from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig - return AzurePassthroughConfig().logging_non_streaming_response( # pyright: ignore[reportUnknownMemberType] # the Azure config still types request_data as a bare dict + chat_result: Final = AzurePassthroughConfig().logging_non_streaming_response( # pyright: ignore[reportUnknownMemberType] # the Azure config still types request_data as a bare dict model=model, custom_llm_provider=custom_llm_provider, httpx_response=httpx_response, @@ -104,6 +134,9 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): logging_obj=logging_obj, endpoint=endpoint, ) + if chat_result is not None: + return chat_result + return StandardPassThroughResponseObject(response=relayed_body(httpx_response)) def handle_logging_collected_chunks( self, diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index 67851d2d58e..e343d05c5df 100644 --- a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -66,6 +66,72 @@ def test_model_inside_the_path_stays_and_query_params_are_forwarded(): assert str(url) == f"{FOUNDRY_BASE}/openai/deployments/gpt-5.4-mini/chat/completions?api-version=2024-10-21" +def test_api_base_that_already_ends_in_models_is_cut_back_to_the_foundry_root(): + url, base = AzureAIPassthroughConfig().get_complete_url( + api_base=f"{FOUNDRY_BASE}/models", + api_key="key", + model="gpt-5.4-mini", + endpoint="gpt-5.4-mini/models/chat/completions", + request_query_params={"api-version": "2024-05-01-preview"}, + litellm_params={}, + ) + + assert str(url) == f"{FOUNDRY_BASE}/models/chat/completions?api-version=2024-05-01-preview" + assert base == FOUNDRY_BASE + + +def test_parse_relay_under_a_models_api_base_targets_the_foundry_root(): + url, _ = AzureAIPassthroughConfig().get_complete_url( + api_base=f"{FOUNDRY_BASE}/models", + api_key="key", + model="Cohere-parse-v5", + endpoint="Cohere-parse-v5/providers/cohere/v2/parse", + request_query_params=None, + litellm_params={}, + ) + + assert str(url) == f"{FOUNDRY_BASE}/providers/cohere/v2/parse" + + +def test_deployment_api_version_fills_in_when_the_caller_sends_none(): + url, _ = AzureAIPassthroughConfig().get_complete_url( + api_base=FOUNDRY_BASE, + api_key="key", + model="gpt-5.4-mini", + endpoint="gpt-5.4-mini/models/chat/completions", + request_query_params=None, + litellm_params={"api_version": "2024-05-01-preview"}, + ) + + assert str(url) == f"{FOUNDRY_BASE}/models/chat/completions?api-version=2024-05-01-preview" + + +def test_callers_api_version_beats_the_deployments(): + url, _ = AzureAIPassthroughConfig().get_complete_url( + api_base=FOUNDRY_BASE, + api_key="key", + model="gpt-5.4-mini", + endpoint="gpt-5.4-mini/models/chat/completions", + request_query_params={"api-version": "2025-04-01-preview"}, + litellm_params={"api_version": "2024-05-01-preview"}, + ) + + assert str(url) == f"{FOUNDRY_BASE}/models/chat/completions?api-version=2025-04-01-preview" + + +def test_api_version_on_the_configured_api_base_is_the_last_fallback(): + url, _ = AzureAIPassthroughConfig().get_complete_url( + api_base=f"{FOUNDRY_BASE}/models/chat/completions?api-version=2024-05-01-preview", + api_key="key", + model="gpt-5.4-mini", + endpoint="gpt-5.4-mini/models/chat/completions", + request_query_params=None, + litellm_params={}, + ) + + assert str(url) == f"{FOUNDRY_BASE}/models/chat/completions?api-version=2024-05-01-preview" + + def test_missing_api_base_raises_instead_of_building_a_relative_url(): with pytest.raises(ValueError, match="AZURE_AI_API_BASE"): AzureAIPassthroughConfig().get_complete_url( @@ -116,7 +182,7 @@ def test_no_credentials_at_all_raises(): @pytest.mark.parametrize( "request_data, expected", - [({"stream": True}, True), ({"stream": False}, False), ({}, False)], + [({"stream": True}, True), ({"stream": 1}, True), ({"stream": False}, False), ({}, False)], ) def test_is_streaming_request_reads_the_stream_flag(request_data, expected): assert AzureAIPassthroughConfig().is_streaming_request(endpoint="models/chat/completions", request_data=request_data) is expected @@ -155,15 +221,14 @@ def test_chat_completions_relay_yields_a_model_response_for_cost_tracking(): assert result.usage.completion_tokens == 8 -def test_non_chat_relay_yields_no_cost_response(): +def _non_chat_logging_result(content: bytes, content_type: str): parse_response = httpx.Response( status_code=200, - headers={"content-type": "application/json"}, - content=b'{"id":"parse-1","pages":[]}', + headers={"content-type": content_type}, + content=content, request=httpx.Request("POST", f"{FOUNDRY_BASE}/providers/cohere/v2/parse"), ) - - result = AzureAIPassthroughConfig().logging_non_streaming_response( + return AzureAIPassthroughConfig().logging_non_streaming_response( model="Cohere-parse-v5", custom_llm_provider="azure_ai", httpx_response=parse_response, @@ -172,7 +237,15 @@ def test_non_chat_relay_yields_no_cost_response(): endpoint="providers/cohere/v2/parse", ) - assert result is None + +def test_non_chat_relay_logs_the_parsed_body_so_spend_tracking_sees_the_call(): + result = _non_chat_logging_result(b'{"id":"parse-1","pages":[],"meta":{"billed_units":{"pages":1}}}', "application/json") + + assert result == {"response": {"id": "parse-1", "pages": [], "meta": {"billed_units": {"pages": 1}}}} + + +def test_non_chat_relay_with_a_non_json_body_logs_the_raw_text(): + assert _non_chat_logging_result(b"page one", "text/plain") == {"response": "page one"} def test_streaming_chat_completion_chunks_are_costed_like_azure(): From 2f981d14e4262db3e573891ba94f0e66209b216d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 02:09:26 -0700 Subject: [PATCH 045/411] refactor(passthrough): inject the streaming prompt-token counter instead of a default-image flag --- .../streaming_chunk_builder_utils.py | 8 ++-- litellm/main.py | 8 ++-- .../openai_passthrough_logging_handler.py | 44 +++++++++++++++++- ruff-strict-budget.json | 4 +- .../test_azure_passthrough_transformation.py | 44 ++++++++++++++++-- ...test_openai_passthrough_logging_handler.py | 46 +++++++++++++++++++ type-discipline-budget.json | 6 +-- 7 files changed, 142 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 1dca64f1a80..f30a8e5f8bd 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1,6 +1,6 @@ import base64 import time -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from itertools import groupby from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypedDict, Union, cast @@ -994,7 +994,7 @@ class ChunkProcessor: completion_output: str, messages: Sequence | None = None, reasoning_tokens: int | None = None, - use_default_image_token_count: bool = False, + count_prompt_tokens: Callable[[], int] | None = None, ) -> Usage: """ Calculate usage for the given chunks. @@ -1019,8 +1019,8 @@ class ChunkProcessor: cost: Final[float | None] = calculated_usage_per_chunk["cost"] try: - returned_usage.prompt_tokens = prompt_tokens or token_counter( - model=model, messages=messages, use_default_image_token_count=use_default_image_token_count + returned_usage.prompt_tokens = prompt_tokens or ( + count_prompt_tokens() if count_prompt_tokens else token_counter(model=model, messages=messages) ) except Exception: # don't allow this failing to block a complete streaming response from being returned print_verbose("token_counter failed, assuming prompt tokens is 0") diff --git a/litellm/main.py b/litellm/main.py index d125f1c1d7f..a3830192e7a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -19,7 +19,7 @@ import random import sys import time import traceback -from collections.abc import AsyncIterator, Coroutine, Iterable, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mapping, Sequence from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy @@ -8674,7 +8674,7 @@ def stream_chunk_builder( start_time=None, end_time=None, logging_obj: Optional["Logging"] = None, - use_default_image_token_count: bool = False, + count_prompt_tokens: Callable[[], int] | None = None, ) -> ModelResponse | TextCompletionResponse | None: try: if chunks is None: @@ -8748,7 +8748,7 @@ def stream_chunk_builder( completion_output=completion_output, messages=messages, reasoning_tokens=0, - use_default_image_token_count=use_default_image_token_count, + count_prompt_tokens=count_prompt_tokens, ) setattr(response, "usage", usage) @@ -8926,7 +8926,7 @@ def stream_chunk_builder( completion_output=completion_output, messages=messages, reasoning_tokens=reasoning_tokens, - use_default_image_token_count=use_default_image_token_count, + count_prompt_tokens=count_prompt_tokens, ) setattr(response, "usage", usage) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index e21105f760c..6a142d08be5 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -13,6 +13,7 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, @@ -97,6 +98,45 @@ def _is_openai_compatible_url(url_route: str | None) -> bool: return False +def _is_remote_high_detail_image(part: object) -> bool: + if not isinstance(part, Mapping) or part.get("type") != "image_url": + return False + image_url: Final = part.get("image_url") + if not isinstance(image_url, Mapping): + return False + url: Final = image_url.get("url") + return isinstance(url, str) and url.startswith(("http://", "https://")) and image_url.get("detail") == "high" + + +def _content_parts(message: Mapping[str, object]) -> Sequence[object]: + content: Final = message.get("content") + return content if isinstance(content, list) else () + + +def _without_remote_high_detail_images(message: Mapping[str, object]) -> Mapping[str, object]: + if not isinstance(message.get("content"), list): + return message + kept_parts: Final = [ # mutable-ok: token_counter reads message content only when it is a list + part for part in _content_parts(message) if not _is_remote_high_detail_image(part) + ] + return {**message, "content": kept_parts} # mutable-ok: token_counter rejects any message that is not a dict + + +def count_relayed_prompt_tokens(model: str, messages: Sequence[Mapping[str, object]] | None) -> int: + if messages is None: + return 0 + remote_high_detail_images: Final = sum( + 1 for message in messages for part in _content_parts(message) if _is_remote_high_detail_image(part) + ) + local_messages: Final = [ # mutable-ok: token_counter takes a list of messages + _without_remote_high_detail_images(message) for message in messages + ] + return ( + litellm.token_counter(model=model, messages=local_messages) + + DEFAULT_IMAGE_TOKEN_COUNT * remote_high_detail_images + ) + + class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): """ OpenAI-specific passthrough logging handler that provides cost tracking for /chat/completions endpoints. @@ -561,7 +601,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): # Build complete response from chunks complete_streaming_response: Final = litellm.stream_chunk_builder( - chunks=all_openai_chunks, messages=messages, use_default_image_token_count=True + chunks=all_openai_chunks, + messages=messages, + count_prompt_tokens=lambda: count_relayed_prompt_tokens(model, messages), ) return complete_streaming_response diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 907317e81b1..574bf01cc34 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -57,7 +57,7 @@ "limit": 3 }, "BLE001": { - "limit": 2910 + "limit": 2904 }, "C401": { "limit": 8 @@ -201,7 +201,7 @@ "limit": 310 }, "SIM103": { - "limit": 113 + "limit": 110 }, "SIM113": { "limit": 3 diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index b07b7915277..5d9e4312b5e 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -2,8 +2,10 @@ import json from unittest.mock import MagicMock import httpx +import pytest import litellm +from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig @@ -159,7 +161,7 @@ def test_azure_passthrough_streaming_chunks_count_remote_image_prompt_tokens_wit "role": "user", "content": [ {"type": "text", "text": "Describe this"}, - {"type": "image_url", "image_url": {"url": "http://127.0.0.1:9/doc.png"}}, + {"type": "image_url", "image_url": {"url": "http://127.0.0.1:9/doc.png", "detail": "high"}}, ], } ] @@ -174,10 +176,10 @@ def test_azure_passthrough_streaming_chunks_count_remote_image_prompt_tokens_wit endpoint="openai/deployments/gpt-4.1-mini/chat/completions", ) + text_only_messages = [{"role": "user", "content": [{"type": "text", "text": "Describe this"}]}] assert isinstance(response, ModelResponse) - assert response.usage.prompt_tokens > 0 - assert response.usage.prompt_tokens == litellm.token_counter( - model="gpt-4.1-mini", messages=messages, use_default_image_token_count=True + assert response.usage.prompt_tokens == ( + litellm.token_counter(model="gpt-4.1-mini", messages=text_only_messages) + DEFAULT_IMAGE_TOKEN_COUNT ) @@ -218,3 +220,37 @@ def test_azure_passthrough_url_prefers_the_deployments_api_version(): ) assert url.params["api-version"] == "2024-10-21" + + +def test_azure_passthrough_url_strips_the_leading_router_model_segment(): + url, _ = AzurePassthroughConfig().get_complete_url( + api_base="https://my-resource.openai.azure.com", + api_key="key", + model="gpt-4.1-mini", + endpoint="gpt-4.1-mini/openai/deployments/gpt-4.1-mini/chat/completions", + request_query_params={"api-version": "2024-10-21"}, + litellm_params={}, + ) + + assert str(url) == "https://my-resource.openai.azure.com/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2024-10-21" + + +def test_azure_passthrough_url_rewrites_the_model_group_only_as_a_whole_segment(): + url, _ = AzurePassthroughConfig().get_complete_url( + api_base="https://my-resource.openai.azure.com", + api_key="key", + model="gpt-4.1-mini", + endpoint="gpt/openai/deployments/gpt-4o/chat/completions", + request_query_params={"api-version": "2024-10-21"}, + litellm_params={"litellm_metadata": {"model_group": "gpt"}}, + ) + + assert str(url) == "https://my-resource.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21" + + +@pytest.mark.parametrize( + "request_data, expected", + [({"stream": True}, True), ({"stream": 1}, True), ({"stream": False}, False), ({}, False)], +) +def test_azure_passthrough_is_streaming_request_reads_the_stream_flag(request_data, expected): + assert AzurePassthroughConfig().is_streaming_request(endpoint="openai/deployments/x/chat/completions", request_data=request_data) is expected diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index 6f9142c85df..23c26a0b8a3 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -8,9 +8,11 @@ import pytest import litellm +from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import ( OpenAIPassthroughLoggingHandler, + count_relayed_prompt_tokens, ) from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, @@ -2037,3 +2039,47 @@ class TestOpenAIPassthroughEmbeddingsSpendLog: if __name__ == "__main__": pytest.main([__file__]) + + +ONE_PIXEL_PNG_DATA_URL = ( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" +) +UNREACHABLE_IMAGE_URL = "http://127.0.0.1:9/doc.png" +TEXT_ONLY_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "Describe this"}]}] + + +def _image_messages(url: str, detail: str) -> list[dict]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this"}, + {"type": "image_url", "image_url": {"url": url, "detail": detail}}, + ], + } + ] + + +def test_count_relayed_prompt_tokens_counts_a_data_url_image_exactly(): + messages = _image_messages(ONE_PIXEL_PNG_DATA_URL, "high") + + assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) == litellm.token_counter( + model="gpt-4.1-mini", messages=messages + ) + + +def test_count_relayed_prompt_tokens_keeps_a_low_detail_remote_image_at_the_base_count(): + messages = _image_messages(UNREACHABLE_IMAGE_URL, "low") + + assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) == litellm.token_counter( + model="gpt-4.1-mini", messages=messages + ) + assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) < DEFAULT_IMAGE_TOKEN_COUNT + + +def test_count_relayed_prompt_tokens_estimates_only_the_remote_high_detail_image(): + messages = _image_messages(UNREACHABLE_IMAGE_URL, "high") + + assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) == ( + litellm.token_counter(model="gpt-4.1-mini", messages=TEXT_ONLY_MESSAGES) + DEFAULT_IMAGE_TOKEN_COUNT + ) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 78cd84f5b35..3a4abfbc20a 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22166 + "limit": 22156 }, "LIT002": { "limit": 26745 @@ -27,10 +27,10 @@ "limit": 0 }, "LIT010": { - "limit": 16446 + "limit": 16434 }, "LIT011": { - "limit": 5506 + "limit": 5504 }, "LIT012": { "limit": 4486 From 4706acef958a726f1407ad5413ea2db896d6c825 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 03:48:06 -0700 Subject: [PATCH 046/411] fix(passthrough): charge remote high-detail images at the high-detail upper bound --- litellm/litellm_core_utils/token_counter.py | 7 ++++++ .../openai_passthrough_logging_handler.py | 4 ++-- ruff-strict-budget.json | 4 ++-- .../litellm_core_utils/test_token_counter.py | 22 ++++++++++++++++++- .../test_azure_passthrough_transformation.py | 4 ++-- ...test_openai_passthrough_logging_handler.py | 8 +++---- type-discipline-budget.json | 6 ++--- 7 files changed, 41 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 3732ffd734c..4c66d878f14 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -172,6 +172,13 @@ def calculate_tiles_needed( return total_tiles +def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int: + largest_tile_count: Final = calculate_tiles_needed( + MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES + ) + return base_tokens + (base_tokens * 2) * largest_tile_count + + def _unpack_ints(fmt: str, buffer: bytes) -> tuple[int, ...]: return struct.unpack(fmt, buffer) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 6a142d08be5..206002d13f7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -13,11 +13,11 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, ) +from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.llms.openai.openai import OpenAIConfig from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -133,7 +133,7 @@ def count_relayed_prompt_tokens(model: str, messages: Sequence[Mapping[str, obje ] return ( litellm.token_counter(model=model, messages=local_messages) - + DEFAULT_IMAGE_TOKEN_COUNT * remote_high_detail_images + + high_detail_image_token_upper_bound() * remote_high_detail_images ) diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 574bf01cc34..134eec578bc 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -57,7 +57,7 @@ "limit": 3 }, "BLE001": { - "limit": 2904 + "limit": 2900 }, "C401": { "limit": 8 @@ -201,7 +201,7 @@ "limit": 310 }, "SIM103": { - "limit": 110 + "limit": 108 }, "SIM113": { "limit": 3 diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 4694fa8fbed..1898fd57220 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1,5 +1,6 @@ #### What this tests #### # This tests litellm.token_counter.token_counter() function +import base64 import importlib import time import traceback @@ -14,7 +15,11 @@ import litellm from litellm import create_pretrained_tokenizer, decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.litellm_core_utils.token_counter import _get_tiktoken_count_function +from litellm.litellm_core_utils.token_counter import ( + _get_tiktoken_count_function, + calculate_img_tokens, + high_detail_image_token_upper_bound, +) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new from tests.large_text import text from tests.test_litellm.litellm_core_utils.messages_with_counts import ( @@ -1412,3 +1417,18 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): assert _count_user_content([prompt, named]) == _count_user_content( [prompt, {"type": "text", "text": "report.pdf"}] ) + + +def _png_data_url(width: int, height: int) -> str: + ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") + return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() + + +@pytest.mark.parametrize(("width", "height"), [(1, 1), (768, 768), (2000, 768), (768, 2000), (4096, 4096), (8000, 3072)]) +def test_high_detail_image_token_upper_bound_covers_every_image_size(width: int, height: int) -> None: + assert calculate_img_tokens(_png_data_url(width, height), mode="high") <= high_detail_image_token_upper_bound() + + +def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_image() -> None: + assert calculate_img_tokens(_png_data_url(2000, 768), mode="high") == high_detail_image_token_upper_bound() + assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound() diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 5d9e4312b5e..8ffd632fe85 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -5,9 +5,9 @@ import httpx import pytest import litellm -from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT +from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig from litellm.types.utils import ModelResponse @@ -179,7 +179,7 @@ def test_azure_passthrough_streaming_chunks_count_remote_image_prompt_tokens_wit text_only_messages = [{"role": "user", "content": [{"type": "text", "text": "Describe this"}]}] assert isinstance(response, ModelResponse) assert response.usage.prompt_tokens == ( - litellm.token_counter(model="gpt-4.1-mini", messages=text_only_messages) + DEFAULT_IMAGE_TOKEN_COUNT + litellm.token_counter(model="gpt-4.1-mini", messages=text_only_messages) + high_detail_image_token_upper_bound() ) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index 23c26a0b8a3..0ddafd20e68 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -8,8 +8,8 @@ import pytest import litellm -from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import ( OpenAIPassthroughLoggingHandler, count_relayed_prompt_tokens, @@ -2074,12 +2074,12 @@ def test_count_relayed_prompt_tokens_keeps_a_low_detail_remote_image_at_the_base assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) == litellm.token_counter( model="gpt-4.1-mini", messages=messages ) - assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) < DEFAULT_IMAGE_TOKEN_COUNT + assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) < high_detail_image_token_upper_bound() -def test_count_relayed_prompt_tokens_estimates_only_the_remote_high_detail_image(): +def test_count_relayed_prompt_tokens_charges_only_the_remote_high_detail_image_at_the_upper_bound(): messages = _image_messages(UNREACHABLE_IMAGE_URL, "high") assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) == ( - litellm.token_counter(model="gpt-4.1-mini", messages=TEXT_ONLY_MESSAGES) + DEFAULT_IMAGE_TOKEN_COUNT + litellm.token_counter(model="gpt-4.1-mini", messages=TEXT_ONLY_MESSAGES) + high_detail_image_token_upper_bound() ) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 3a4abfbc20a..b4c17ffd47b 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22156 + "limit": 22146 }, "LIT002": { "limit": 26745 @@ -27,10 +27,10 @@ "limit": 0 }, "LIT010": { - "limit": 16434 + "limit": 16422 }, "LIT011": { - "limit": 5504 + "limit": 5502 }, "LIT012": { "limit": 4486 From bbf51146ca7ffa63e5a46c903fa13b14b7f48c59 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 5 Sep 2026 13:29:38 +0000 Subject: [PATCH 047/411] fix(pricing): correct Vertex Haiku 4.5 output limit, Bedrock Mantle gpt-oss rates, add Scaleway deprecation dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 32 +++++++++++-------- model_prices_and_context_window.json | 32 +++++++++++-------- .../test_bedrock_mantle_transformation.py | 4 +-- 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2459ed940e0..1bc4c275161 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45421,8 +45421,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -45446,8 +45446,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51727,7 +51727,8 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-10-01" }, "scaleway/openai/gpt-oss-120b": { "input_cost_per_token": 1.5e-07, @@ -51766,7 +51767,8 @@ "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-08-01" }, "scaleway/hcompany/holo2-30b-a3b": { "input_cost_per_token": 3e-07, @@ -51777,7 +51779,8 @@ "mode": "chat", "output_cost_per_token": 7e-07, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-08-09" }, "scaleway/mistralai/mistral-medium-3.5-128b": { "input_cost_per_token": 1.5e-06, @@ -51800,7 +51803,8 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-06, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-08-01" }, "scaleway/mistralai/voxtral-small-24b-2507": { "input_cost_per_audio_token": 1.5e-07, @@ -51811,7 +51815,8 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.5e-07, - "supports_audio_input": true + "supports_audio_input": true, + "deprecation_date": "2026-08-01" }, "scaleway/mistralai/mistral-small-3.2-24b-instruct-2506": { "input_cost_per_token": 1.5e-07, @@ -51833,7 +51838,8 @@ "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-10-01" }, "scaleway/BAAI/bge-multilingual-gemma2": { "input_cost_per_token": 1e-07, @@ -54283,7 +54289,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { - "input_cost_per_token": 7.5e-08, + "input_cost_per_token": 7e-08, "output_cost_per_token": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, @@ -54317,8 +54323,8 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 65536, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2459ed940e0..1bc4c275161 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45421,8 +45421,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -45446,8 +45446,8 @@ "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 64000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51727,7 +51727,8 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-10-01" }, "scaleway/openai/gpt-oss-120b": { "input_cost_per_token": 1.5e-07, @@ -51766,7 +51767,8 @@ "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-08-01" }, "scaleway/hcompany/holo2-30b-a3b": { "input_cost_per_token": 3e-07, @@ -51777,7 +51779,8 @@ "mode": "chat", "output_cost_per_token": 7e-07, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-08-09" }, "scaleway/mistralai/mistral-medium-3.5-128b": { "input_cost_per_token": 1.5e-06, @@ -51800,7 +51803,8 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-06, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-08-01" }, "scaleway/mistralai/voxtral-small-24b-2507": { "input_cost_per_audio_token": 1.5e-07, @@ -51811,7 +51815,8 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.5e-07, - "supports_audio_input": true + "supports_audio_input": true, + "deprecation_date": "2026-08-01" }, "scaleway/mistralai/mistral-small-3.2-24b-instruct-2506": { "input_cost_per_token": 1.5e-07, @@ -51833,7 +51838,8 @@ "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, - "supports_function_calling": true + "supports_function_calling": true, + "deprecation_date": "2026-10-01" }, "scaleway/BAAI/bge-multilingual-gemma2": { "input_cost_per_token": 1e-07, @@ -54283,7 +54289,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { - "input_cost_per_token": 7.5e-08, + "input_cost_per_token": 7e-08, "output_cost_per_token": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, @@ -54317,8 +54323,8 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 3e-07, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 65536, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index b88c27e64b9..92c012d339c 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -696,8 +696,8 @@ class TestBedrockMantlePricing: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") litellm.add_known_models() info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-20b") - # Bedrock pricing: $0.075/M input, $0.30/M output - assert info["input_cost_per_token"] == pytest.approx(7.5e-8) + # Bedrock pricing: $0.07/M input, $0.30/M output + assert info["input_cost_per_token"] == pytest.approx(7e-8) assert info["output_cost_per_token"] == pytest.approx(3e-7) def test_pricing_significantly_cheaper_than_openai_native(self, monkeypatch): From 528f3ae0161192004ca76d615363c748dcfdcd0d Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 5 Sep 2026 13:46:46 +0000 Subject: [PATCH 048/411] test(bedrock_mantle): cover gpt-oss-safeguard-20b input and output rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock_mantle/test_bedrock_mantle_transformation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 92c012d339c..b3eec97058a 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -700,6 +700,14 @@ class TestBedrockMantlePricing: assert info["input_cost_per_token"] == pytest.approx(7e-8) assert info["output_cost_per_token"] == pytest.approx(3e-7) + def test_gpt_oss_safeguard_20b_pricing(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.add_known_models() + info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-safeguard-20b") + # Bedrock pricing: $0.07/M input, $0.20/M output + assert info["input_cost_per_token"] == pytest.approx(7e-8) + assert info["output_cost_per_token"] == pytest.approx(2e-7) + def test_pricing_significantly_cheaper_than_openai_native(self, monkeypatch): """ Verify Bedrock Mantle pricing is cheaper than OpenAI's direct API pricing. From b4e7776ab3534b4c6b91073aea1a1bbf8f681e83 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 5 Sep 2026 19:21:18 +0000 Subject: [PATCH 049/411] fix(model_prices): add gemini lyria-3.5, voyage-multilingual-2, chatgpt gpt-5.5 and gpt-5.6 entries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 99 +++++++++++++++++++ model_prices_and_context_window.json | 99 +++++++++++++++++++ .../test_chatgpt_responses_transformation.py | 53 +++++++++- 3 files changed, 249 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1bc4c275161..8780c41b70a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27541,6 +27541,70 @@ "max_tokens": 8191, "mode": "embedding" }, + "chatgpt/gpt-5.5": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.5", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-luna": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-luna", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-sol": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-sol", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-terra": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-terra", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", "max_input_tokens": 1050000, @@ -47654,6 +47718,16 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-multilingual-2": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-3-large": { "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", @@ -60044,6 +60118,31 @@ "supports_web_search": false, "output_cost_per_image": 0.08 }, + "gemini/lyria-3.5": { + "input_cost_per_token": 0, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 0, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "output_cost_per_image": 0.08 + }, "perplexity/anthropic/claude-fable-5": { "litellm_provider": "perplexity", "mode": "responses", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1bc4c275161..8780c41b70a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27541,6 +27541,70 @@ "max_tokens": 8191, "mode": "embedding" }, + "chatgpt/gpt-5.5": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.5", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-luna": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-luna", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-sol": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-sol", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "chatgpt/gpt-5.6-terra": { + "litellm_provider": "chatgpt", + "source": "https://platform.openai.com/docs/models/gpt-5.6-terra", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", "max_input_tokens": 1050000, @@ -47654,6 +47718,16 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, + "voyage/voyage-multilingual-2": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-3-large": { "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", @@ -60044,6 +60118,31 @@ "supports_web_search": false, "output_cost_per_image": 0.08 }, + "gemini/lyria-3.5": { + "input_cost_per_token": 0, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 0, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "output_cost_per_image": 0.08 + }, "perplexity/anthropic/claude-fable-5": { "litellm_provider": "perplexity", "mode": "responses", diff --git a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 8e0415d50de..8a0027fdb53 100644 --- a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -10,18 +10,23 @@ from unittest.mock import MagicMock, patch import httpx import pytest - +import litellm +from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig from litellm.llms.openai.common_utils import OpenAIError +from litellm.main import responses_api_bridge_check from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager -from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig class TestChatGPTResponsesAPITransformation: @pytest.mark.parametrize( "model_name", [ + "chatgpt/gpt-5.5", + "chatgpt/gpt-5.6-luna", + "chatgpt/gpt-5.6-sol", + "chatgpt/gpt-5.6-terra", "chatgpt/gpt-5.4", "chatgpt/gpt-5.4-pro", "chatgpt/gpt-5.3-chat-latest", @@ -40,6 +45,50 @@ class TestChatGPTResponsesAPITransformation: assert isinstance(config, ChatGPTResponsesAPIConfig) assert config.custom_llm_provider == LlmProviders.CHATGPT + @pytest.mark.parametrize( + "model_name", + [ + "chatgpt/gpt-5.5", + "chatgpt/gpt-5.6-luna", + "chatgpt/gpt-5.6-sol", + "chatgpt/gpt-5.6-terra", + ], + ) + def test_chatgpt_responses_model_metadata(self, model_name, local_model_cost_map): + model_info = litellm.get_model_info(model_name) + + assert model_info["litellm_provider"] == "chatgpt" + assert model_info["mode"] == "responses" + assert model_info["supported_endpoints"] == [ + "/v1/chat/completions", + "/v1/responses", + ] + assert model_info["max_input_tokens"] == 1050000 + assert model_info["max_output_tokens"] == 128000 + + @pytest.mark.parametrize( + "model_name", + [ + "gpt-5.5", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + ], + ) + def test_chatgpt_models_bridge_chat_completions_to_responses(self, model_name, local_model_cost_map): + """A chat completions request for these models must take the Responses bridge. + + `gpt-5.6-*` also exists as an openai chat model, so an unregistered + chatgpt model resolves to mode "chat" here and never reaches the bridge. + """ + model_info, resolved_model = responses_api_bridge_check( + model=model_name, + custom_llm_provider="chatgpt", + ) + + assert model_info["mode"] == "responses" + assert resolved_model == model_name + @patch("litellm.llms.chatgpt.responses.transformation.Authenticator") def test_chatgpt_responses_endpoint_url(self, mock_authenticator_class): mock_auth_instance = MagicMock() From 7015bf37bb3830dab79b3efd886eee9b5b7ffad6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 17:32:17 -0700 Subject: [PATCH 050/411] fix(proxy): apply team model aliases on the JWT auth path Team reads on the auth path never loaded the team's alias table, and the team-based JWT branch copied a hand-picked subset of team fields onto UserAPIKeyAuth, so aliases (and a few other team grants) never reached JWT callers: restricted teams 403'd alias requests and open teams 400'd them Load the alias relation where the team row is read and cached, project the team onto every team_* token field through one shared team_grants helper used by both JWT returns, and keep the relation when team model add/delete rewrites the cached team. ui_sso reuses the shared alias table model Resolves LIT-5858 Claude-Session: https://claude.ai/code/session_01EX13mWex6RaBo9PYnkAtFW --- basedpyright-code-budget.json | 2 +- litellm/proxy/auth/auth_checks.py | 9 +- litellm/proxy/auth/handle_jwt.py | 5 +- litellm/proxy/auth/team_grants.py | 122 +++++++++++++++++ litellm/proxy/auth/user_api_key_auth.py | 22 +-- .../management_endpoints/team_endpoints.py | 4 +- litellm/proxy/management_endpoints/ui_sso.py | 23 +--- .../proxy/auth/test_auth_checks.py | 64 +++++++++ .../proxy/auth/test_handle_jwt.py | 52 +++++++ .../proxy/auth/test_team_grants.py | 129 ++++++++++++++++++ .../proxy/auth/test_user_api_key_auth.py | 116 ++++++++++++++++ .../test_team_endpoints.py | 44 ++++++ type-discipline-budget.json | 2 +- 13 files changed, 547 insertions(+), 47 deletions(-) create mode 100644 litellm/proxy/auth/team_grants.py create mode 100644 tests/test_litellm/proxy/auth/test_team_grants.py diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 0b0a61192e6..8bdc251c684 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -105,7 +105,7 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38271 + "limit": 38269 }, "reportUnknownParameterType": { "limit": 19584 diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index dc693317de0..a71a1993064 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -475,6 +475,7 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None _NO_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({}) +_TEAM_GRANT_RELATIONS: Final[Mapping[str, object]] = MappingProxyType({"litellm_model_table": True}) def _has_ptu_flat_cost(model: str, llm_router: "Router") -> bool: @@ -2858,7 +2859,9 @@ class TeamNotFoundError(HTTPException): async def _get_team_db_check( team_id: str, prisma_client: PrismaClient, team_id_upsert: bool | None = None ) -> "_PrismaTeamRow | None": - response = await _team_table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) + response = await _team_table(TeamRepository(prisma_client)).find_unique( + where={"team_id": team_id}, include=_TEAM_GRANT_RELATIONS + ) if response is None and team_id_upsert: from litellm.proxy.management_endpoints.team_endpoints import new_team @@ -3158,7 +3161,9 @@ async def get_team_object_by_alias( # Query database by team_alias try: - teams: Final = await _team_table(TeamRepository(prisma_client)).find_many(where={"team_alias": team_alias}) + teams: Final = await _team_table(TeamRepository(prisma_client)).find_many( + where={"team_alias": team_alias}, include=_TEAM_GRANT_RELATIONS + ) if not teams: raise HTTPException( diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 0795cee7409..69091ee8344 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -53,6 +53,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import can_team_access_model from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.auth.team_grants import team_model_aliases from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, get_management_object_ttl, @@ -1595,7 +1596,7 @@ class JWTAuthManager: model=requested_model, team_object=team_object, llm_router=llm_router, - team_model_aliases=None, + team_model_aliases=team_model_aliases(team_object), ) ): is_allowed = allowed_routes_check( @@ -2132,7 +2133,7 @@ class JWTAuthManager: model=requested_model, team_object=team_object, llm_router=llm_router, - team_model_aliases=None, + team_model_aliases=team_model_aliases(team_object), ) except ProxyException: continue diff --git a/litellm/proxy/auth/team_grants.py b/litellm/proxy/auth/team_grants.py new file mode 100644 index 00000000000..1196011dcdd --- /dev/null +++ b/litellm/proxy/auth/team_grants.py @@ -0,0 +1,122 @@ +"""Project a team row (plus the caller's membership in it) onto the ``team_*`` fields of ``UserAPIKeyAuth``. + +The virtual-key path gets these fields for free from the combined-view SQL join. Every other auth path +starts from a ``LiteLLM_TeamTable`` object instead and has to copy them over by hand, which is how JWT +callers kept losing grants (aliases, permissions, limits) one field at a time. Build the badge through +``team_grants`` and the two paths cannot drift. +""" + +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Annotated, Final + +from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError +from pydantic.main import IncEx +from typing_extensions import ReadOnly, TypedDict + +from litellm.proxy._types import ( + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, + Member, +) + +_MODEL_ALIASES_ADAPTER: Final = TypeAdapter(dict[str, str]) +_JSON_COLUMNS: Final[Mapping[str, IncEx | bool]] = MappingProxyType( + {"metadata": True, "litellm_model_table": MappingProxyType({"model_aliases": True})} +) + + +def _decode_model_aliases(value: object) -> object: + """``LiteLLM_ModelTable.model_aliases`` is typed ``str | dict``; writers hand Prisma ``json.dumps(...)``, so take both.""" + if not isinstance(value, str): + return value + try: + return _MODEL_ALIASES_ADAPTER.validate_json(value) + except ValidationError: + return None + + +class TeamModelAliasTable(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + + model_aliases: Annotated[Mapping[str, str] | None, BeforeValidator(_decode_model_aliases)] = None + + +class _TeamJsonColumns(BaseModel): + """The two loosely typed columns on ``LiteLLM_TeamTable``, re-read with the shape the badge needs.""" + + metadata: Mapping[str, object] | None = None + litellm_model_table: TeamModelAliasTable | None = None + + +class TeamGrants(TypedDict, total=False): + """Keyword arguments for ``UserAPIKeyAuth``. Empty when the caller has no team, so the model's own defaults apply.""" + + team_alias: ReadOnly[str | None] + team_tpm_limit: ReadOnly[int | None] + team_rpm_limit: ReadOnly[int | None] + team_max_budget: ReadOnly[float | None] + team_soft_budget: ReadOnly[float | None] + team_spend: ReadOnly[float | None] + team_models: ReadOnly[Sequence[str]] + team_blocked: ReadOnly[bool] + team_metadata: ReadOnly[Mapping[str, object] | None] + team_model_aliases: ReadOnly[Mapping[str, str] | None] + team_object_permission_id: ReadOnly[str | None] + team_object_permission: ReadOnly[LiteLLM_ObjectPermissionTable | None] + team_member: ReadOnly[Member | None] + team_member_spend: ReadOnly[float | None] + team_member_tpm_limit: ReadOnly[int | None] + team_member_rpm_limit: ReadOnly[int | None] + + +def _json_columns(team_object: LiteLLM_TeamTable) -> _TeamJsonColumns: + try: + return _TeamJsonColumns.model_validate(team_object.model_dump(include=_JSON_COLUMNS)) + except ValidationError: + return _TeamJsonColumns() + + +def team_model_aliases(team_object: LiteLLM_TeamTable | None) -> Mapping[str, str] | None: + if team_object is None: + return None + alias_table: Final = _json_columns(team_object).litellm_model_table + return alias_table.model_aliases if alias_table is not None else None + + +def team_grants( + team_object: LiteLLM_TeamTable | None, + team_membership: LiteLLM_TeamMembership | None, + user_id: str | None, +) -> TeamGrants: + if team_object is None: + return TeamGrants() + json_columns: Final = _json_columns(team_object) + return TeamGrants( + team_alias=team_object.team_alias, + team_tpm_limit=team_object.tpm_limit, + team_rpm_limit=team_object.rpm_limit, + team_max_budget=team_object.max_budget, + team_soft_budget=team_object.soft_budget, + team_spend=team_object.spend, + team_models=tuple(team_object.models), + team_blocked=team_object.blocked, + team_metadata=json_columns.metadata, + team_model_aliases=( + json_columns.litellm_model_table.model_aliases if json_columns.litellm_model_table is not None else None + ), + team_object_permission_id=team_object.object_permission_id, + team_object_permission=team_object.object_permission, + team_member=next( + (m for m in team_object.members_with_roles if user_id is not None and m.user_id == user_id), + None, + ), + team_member_spend=team_membership.spend if team_membership is not None else None, + team_member_tpm_limit=( + team_membership.safe_get_team_member_tpm_limit() if team_membership is not None else None + ), + team_member_rpm_limit=( + team_membership.safe_get_team_member_rpm_limit() if team_membership is not None else None + ), + ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 93293db24c6..08dbe2508ec 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -82,6 +82,7 @@ from litellm.proxy.auth.oauth2_proxy_hook import handle_oauth2_proxy_request from litellm.proxy.auth.resolvers import CredentialRef, Principal from litellm.proxy.auth.resolvers.store import IdentityStore from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.auth.team_grants import team_grants from litellm.proxy.auth.trusted_proxy_utils import get_trusted_proxy_cidrs from litellm.proxy.common_utils.cache_coordinator import EventDrivenCacheCoordinator from litellm.proxy.common_utils.http_parsing_utils import ( @@ -1476,24 +1477,16 @@ async def _user_api_key_auth_builder( user_id=user_id, user_email=user_email, team_id=team_id, - team_alias=(team_object.team_alias if team_object is not None else None), - team_tpm_limit=(team_object.tpm_limit if team_object is not None else None), - team_rpm_limit=(team_object.rpm_limit if team_object is not None else None), - team_models=(team_object.models if team_object is not None else []), - team_metadata=(team_object.metadata if team_object is not None else None), org_id=org_id, end_user_id=end_user_id, parent_otel_span=parent_otel_span, jwt_claims=jwt_claims, + **team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id), ) valid_token = UserAPIKeyAuth( api_key=None, team_id=team_id, - team_alias=(team_object.team_alias if team_object is not None else None), - team_tpm_limit=(team_object.tpm_limit if team_object is not None else None), - team_rpm_limit=(team_object.rpm_limit if team_object is not None else None), - team_models=(team_object.models if team_object is not None else []), user_role=( LitellmUserRoles(user_object.user_role) if user_object is not None and user_object.user_role is not None @@ -1507,17 +1500,8 @@ async def _user_api_key_auth_builder( user_tpm_limit=(user_object.tpm_limit if user_object is not None else None), user_rpm_limit=(user_object.rpm_limit if user_object is not None else None), user_model_max_budget=(user_object.model_max_budget if user_object is not None else None), - team_member_rpm_limit=( - team_membership.safe_get_team_member_rpm_limit() if team_membership is not None else None - ), - team_member_tpm_limit=( - team_membership.safe_get_team_member_tpm_limit() if team_membership is not None else None - ), - team_metadata=(team_object.metadata if team_object is not None else None), jwt_claims=jwt_claims, - ) - valid_token.team_object_permission = ( - team_object.object_permission if team_object is not None else None + **team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id), ) # AUTO_REGISTER deferred from _resolve_jwt_to_virtual_key. diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 2ec68a10f65..c050368b3fe 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -5601,7 +5601,7 @@ async def team_model_add( updated_team: Final = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"updated_at": datetime.now(timezone.utc)}, - include={"object_permission": True}, + include={"litellm_model_table": True, "object_permission": True}, ) if updated_team is None: raise HTTPException( @@ -5688,7 +5688,7 @@ async def team_model_delete( updated_team: Final = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"models": updated_models}, - include={"object_permission": True}, + include={"litellm_model_table": True, "object_permission": True}, ) if updated_team is None: raise HTTPException( diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 3e6434a5afd..c60888e298f 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -22,7 +22,6 @@ from html import escape from types import MappingProxyType from typing import ( TYPE_CHECKING, - Annotated, Any, Final, Literal, @@ -42,7 +41,7 @@ if TYPE_CHECKING: import jwt from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status from fastapi.responses import RedirectResponse -from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -95,6 +94,7 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.auth.team_grants import TeamModelAliasTable from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.admin_ui_utils import ( admin_ui_disabled, @@ -209,31 +209,14 @@ def _team_detail_db(repo: TeamRepository) -> "TableActions[_TeamDetailRow]": return repo.table -_MODEL_ALIASES_ADAPTER: Final = TypeAdapter(dict[str, str]) _SSO_TOKEN_CLAIMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -def _decode_model_aliases(value: object) -> object: - """``/team/new`` stores team model aliases as a JSON-encoded string in the Json column.""" - if not isinstance(value, str): - return value - try: - return _MODEL_ALIASES_ADAPTER.validate_json(value) - except ValidationError: - return None - - -class _TeamModelAliasTable(BaseModel): - model_config = ConfigDict(protected_namespaces=()) - - model_aliases: Annotated[Mapping[str, str] | None, BeforeValidator(_decode_model_aliases)] = None - - class _TeamRowGrants(BaseModel): team_id: str team_alias: str | None = None models: tuple[str, ...] = () - litellm_model_table: _TeamModelAliasTable | None = None + litellm_model_table: TeamModelAliasTable | None = None class CliSsoTeamDetail(BaseModel): diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 2284a05b2e9..5bfef2b6445 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -2374,6 +2374,44 @@ def _mock_prisma_for_team_lookup(find_unique): return mock_prisma_client +_TEAM_ALIAS_TABLE_ROW = {"id": 1, "model_aliases": '{"fast": "gpt-4o"}', "created_by": "admin", "updated_by": "admin"} + + +def _prisma_team_row(include): + """Mimics Prisma: the `litellm_model_table` relation rides on the row only when the query `include`s it.""" + columns = {"team_id": "team-aliases", "team_alias": "aliases", "models": ["gpt-4o"]} + row = ( + {**columns, "litellm_model_table": _TEAM_ALIAS_TABLE_ROW} + if (include or {}).get("litellm_model_table") + else columns + ) + return SimpleNamespace(dict=lambda: row, model_dump=lambda: row) + + +@pytest.mark.asyncio +async def test_get_team_object_loads_model_aliases_relation(): + """LIT-5858: the auth path read teams without `include`ing `litellm_model_table`, so every JWT + team came back with `model_aliases=None` and alias requests 403'd.""" + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.auth.team_grants import team_model_aliases + + async def find_unique(where, include=None): + return _prisma_team_row(include) + + mock_cache = MagicMock() + mock_cache.async_get_cache = AsyncMock(return_value=None) + mock_cache.async_set_cache = AsyncMock() + + team = await get_team_object( + team_id="team-aliases", + prisma_client=_mock_prisma_for_team_lookup(AsyncMock(side_effect=find_unique)), + user_api_key_cache=mock_cache, + check_db_only=True, + ) + + assert team_model_aliases(team) == {"fast": "gpt-4o"} + + @pytest.mark.asyncio async def test_get_team_object_distinguishes_absent_team_from_unreadable_row(): """A deleted team and a database that would not answer both surface as a 404, @@ -6195,6 +6233,32 @@ async def test_get_team_object_by_alias_db_fetch_returns_cached_obj(): assert result.models == ["gpt-4"] +@pytest.mark.asyncio +async def test_get_team_object_by_alias_loads_model_aliases_relation(): + """LIT-5858: same regression as `test_get_team_object_loads_model_aliases_relation`, for the + `team_alias_jwt_field` lookup.""" + from litellm.proxy.auth.auth_checks import get_team_object_by_alias + from litellm.proxy.auth.team_grants import team_model_aliases + + async def find_many(where, include=None): + return [_prisma_team_row(include)] + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_teamtable.find_many = AsyncMock(side_effect=find_many) + + mock_cache = MagicMock() + mock_cache.async_get_cache = AsyncMock(return_value=None) + mock_cache.async_set_cache = AsyncMock() + + team = await get_team_object_by_alias( + team_alias="aliases", + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + + assert team_model_aliases(team) == {"fast": "gpt-4o"} + + @pytest.mark.asyncio async def test_get_org_object_by_alias_db_fetch_returns_validated_org(): from litellm.proxy._types import LiteLLM_OrganizationTable diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index 99a0a4c0a8b..94226b5404d 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -13,6 +13,7 @@ from litellm.proxy._types import ( DEFAULT_JWKS_STALE_TTL, JWTLiteLLMRoleMap, LiteLLM_JWTAuth, + LiteLLM_ModelTable, LiteLLM_TeamMembership, LiteLLM_TeamTable, LiteLLM_UserTable, @@ -1255,6 +1256,57 @@ async def test_find_team_with_model_access_model_group(monkeypatch): assert team_obj.team_id == "team-1" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_aliases", + ['{"fast": "gpt-4o"}', {"fast": "gpt-4o"}], + ids=["json-string", "dict"], +) +async def test_find_team_with_model_access_resolves_team_model_alias(monkeypatch, model_aliases): + """LIT-5858: a JWT team that grants `gpt-4o` under the alias `fast` must resolve a request + for `fast`. The JWT path used to pass `team_model_aliases=None`, so every alias request 403'd.""" + import sys + import types + + from litellm.caching import DualCache + from litellm.proxy.utils import ProxyLogging + from litellm.router import Router + + router = Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}]) + proxy_server_module = types.ModuleType("proxy_server") + proxy_server_module.llm_router = router + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", proxy_server_module) + + team = LiteLLM_TeamTable( + team_id="team-aliases", + models=["gpt-4o"], + litellm_model_table=LiteLLM_ModelTable(model_aliases=model_aliases, created_by="admin", updated_by="admin"), + ) + + async def mock_get_team_object(*args, **kwargs): + return team + + monkeypatch.setattr("litellm.proxy.auth.handle_jwt.get_team_object", mock_get_team_object) + + jwt_handler = JWTHandler() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() + user_api_key_cache = DualCache() + + team_id, team_obj = await JWTAuthManager.find_team_with_model_access( + team_ids={"team-aliases"}, + requested_model="fast", + route="/chat/completions", + jwt_handler=jwt_handler, + prisma_client=None, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=user_api_key_cache), + ) + + assert team_id == "team-aliases" + assert team_obj is team + + @pytest.mark.asyncio async def test_find_team_with_model_access_v1_messages_default_routes(monkeypatch): """Regression for #31189: a single-team JWT that grants the requested model diff --git a/tests/test_litellm/proxy/auth/test_team_grants.py b/tests/test_litellm/proxy/auth/test_team_grants.py new file mode 100644 index 00000000000..447fc1c93a1 --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_team_grants.py @@ -0,0 +1,129 @@ +import pytest + +from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, + LiteLLM_VerificationTokenView, + Member, + UserAPIKeyAuth, +) +from litellm.models.team import LiteLLM_ModelTable +from litellm.proxy.auth.team_grants import team_grants, team_model_aliases + +TEAM_ID = "team-grants" +USER_ID = "user-in-team" +ALIASES = {"fast": "gpt-4o-mini", "smart": "gpt-4o"} + + +def _alias_table(model_aliases) -> LiteLLM_ModelTable: + return LiteLLM_ModelTable(model_aliases=model_aliases, created_by="admin", updated_by="admin") + + +def _full_team(model_aliases=ALIASES) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=TEAM_ID, + team_alias="grants-team", + tpm_limit=1000, + rpm_limit=10, + max_budget=50.0, + soft_budget=25.0, + spend=12.5, + models=["gpt-4o", "gpt-4o-mini"], + blocked=True, + metadata={"tier": "gold"}, + litellm_model_table=_alias_table(model_aliases), + object_permission_id="op-1", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-1", mcp_servers=["mcp-a"]), + members_with_roles=[ + Member(user_id="someone-else", role="user"), + Member(user_id=USER_ID, role="admin"), + ], + ) + + +def _membership() -> LiteLLM_TeamMembership: + return LiteLLM_TeamMembership( + user_id=USER_ID, + team_id=TEAM_ID, + spend=3.25, + litellm_budget_table=LiteLLM_BudgetTable(tpm_limit=500, rpm_limit=5), + ) + + +def test_team_grants_cover_every_team_field_the_key_path_gets(): + """Class guard for LIT-5858 and its siblings: every ``team_*`` column the combined-view SQL hands the + virtual-key path must come out of the projection too, with the team's actual value, so adding a column + to ``LiteLLM_VerificationTokenView`` without teaching ``team_grants`` fails here instead of in prod.""" + team = _full_team() + grants = team_grants(team_object=team, team_membership=_membership(), user_id=USER_ID) + token = UserAPIKeyAuth(team_id=TEAM_ID, **grants) + + view_team_fields = {name for name in LiteLLM_VerificationTokenView.model_fields if name.startswith("team_")} + assert view_team_fields - {"team_id"} <= set(grants) + assert all(grants[name] is not None for name in view_team_fields - {"team_id"}) + + assert token.team_alias == "grants-team" + assert token.team_tpm_limit == 1000 + assert token.team_rpm_limit == 10 + assert token.team_max_budget == 50.0 + assert token.team_soft_budget == 25.0 + assert token.team_spend == 12.5 + assert token.team_models == ["gpt-4o", "gpt-4o-mini"] + assert token.team_blocked is True + assert token.team_metadata == {"tier": "gold"} + assert token.team_model_aliases == ALIASES + assert token.team_object_permission_id == "op-1" + assert token.team_object_permission is not None + assert token.team_object_permission.mcp_servers == ["mcp-a"] + assert token.team_member == Member(user_id=USER_ID, role="admin") + assert token.team_member_spend == 3.25 + assert token.team_member_tpm_limit == 500 + assert token.team_member_rpm_limit == 5 + + +def test_team_grants_without_team_leave_token_defaults(): + token = UserAPIKeyAuth(**team_grants(team_object=None, team_membership=None, user_id=USER_ID)) + assert token == UserAPIKeyAuth() + + +@pytest.mark.parametrize( + "stored_aliases", + [ALIASES, '{"fast": "gpt-4o-mini", "smart": "gpt-4o"}'], + ids=["json-object", "json-string-as-written-by-team-new"], +) +def test_team_model_aliases_decode_both_storage_shapes(stored_aliases): + team = _full_team(model_aliases=stored_aliases) + assert team_model_aliases(team) == ALIASES + assert team_grants(team_object=team, team_membership=None, user_id=None)["team_model_aliases"] == ALIASES + + +@pytest.mark.parametrize("stored_aliases", [None, "not json", '["a", "b"]', {"fast": 3}], ids=str) +def test_team_model_aliases_treat_unusable_column_as_no_aliases(stored_aliases): + team = _full_team(model_aliases=stored_aliases) + assert team_model_aliases(team) is None + assert team_grants(team_object=team, team_membership=None, user_id=None)["team_model_aliases"] is None + + +def test_team_model_aliases_none_without_relation_loaded(): + team = _full_team() + team.litellm_model_table = None + assert team_model_aliases(team) is None + assert team_model_aliases(None) is None + + +def test_team_member_is_the_callers_row_only(): + team = _full_team() + assert team_grants(team_object=team, team_membership=None, user_id="someone-else")["team_member"] == Member( + user_id="someone-else", role="user" + ) + assert team_grants(team_object=team, team_membership=None, user_id="stranger")["team_member"] is None + assert team_grants(team_object=team, team_membership=None, user_id=None)["team_member"] is None + + +def test_membership_limits_absent_without_membership_row(): + grants = team_grants(team_object=_full_team(), team_membership=None, user_id=USER_ID) + assert grants["team_member_spend"] is None + assert grants["team_member_tpm_limit"] is None + assert grants["team_member_rpm_limit"] is None diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index d44f96d95bf..78ffbb0db23 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -6872,3 +6872,119 @@ class TestLitellmReceivedAtStamping: assert result == earlier assert request.state.litellm_received_at == earlier + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_proxy_admin", [False, True], ids=["standard-return", "proxy-admin-return"]) +async def test_jwt_builder_returns_every_team_grant_the_key_path_gets(is_proxy_admin): + """LIT-5858: the team-based JWT path hand-built ``UserAPIKeyAuth`` from a short list of team fields, so the + team's model aliases (and on the admin return, its object permission) never reached the token and alias + requests 403'd. Both returns now go through ``team_grants``; pin the fields that used to be dropped.""" + import litellm.proxy.proxy_server as _proxy_server_mod + from fastapi import Request + from starlette.datastructures import URL + + from litellm.models.team import LiteLLM_ModelTable + from litellm.proxy._types import ( + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, + Member, + ) + + class _AcceptEveryJwt(JWTHandler): + def is_jwt(self, token: str) -> bool: + return True + + jwt_handler = _AcceptEveryJwt() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() + + team = LiteLLM_TeamTable( + team_id="team-jwt-aliases", + team_alias="jwt-aliases", + models=["gpt-4o"], + max_budget=40.0, + spend=4.0, + blocked=False, + metadata={"tier": "gold"}, + litellm_model_table=LiteLLM_ModelTable( + model_aliases='{"fast": "gpt-4o"}', created_by="admin", updated_by="admin" + ), + object_permission_id="op-jwt", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-jwt", mcp_servers=["mcp-a"]), + members_with_roles=[Member(user_id="jwt-user", role="admin")], + ) + membership = LiteLLM_TeamMembership(user_id="jwt-user", team_id="team-jwt-aliases", spend=1.5) + builder_result = { + "is_proxy_admin": is_proxy_admin, + "team_object": team, + "user_object": None, + "end_user_object": None, + "org_object": None, + "token": "jwt", + "team_id": "team-jwt-aliases", + "user_id": "jwt-user", + "user_email": "jwt-user@example.com", + "end_user_id": None, + "org_id": None, + "team_membership": membership, + "jwt_claims": {"sub": "jwt-user"}, + } + + mock_proxy_logging_obj = MagicMock() + mock_proxy_logging_obj.internal_usage_cache = MagicMock() + mock_proxy_logging_obj.internal_usage_cache.dual_cache = AsyncMock() + mock_proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + attrs = { + "prisma_client": MagicMock(), + "user_api_key_cache": DualCache(), + "proxy_logging_obj": mock_proxy_logging_obj, + "master_key": "sk-master-key", + "general_settings": {"enable_jwt_auth": True}, + "llm_model_list": [], + "llm_router": None, + "open_telemetry_logger": None, + "model_max_budget_limiter": MagicMock(), + "user_custom_auth": None, + "jwt_handler": jwt_handler, + "premium_user": True, + "litellm_proxy_admin_name": "admin", + } + originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} + try: + for k, v in attrs.items(): + setattr(_proxy_server_mod, k, v) + request = Request(scope={"type": "http", "headers": [], "method": "POST"}) + request._url = URL(url="/chat/completions") + with patch( # test-quality-ok: auth_builder is the claim-resolution seam; the regression is how its result is projected onto the token + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=builder_result, + ): + token = await _user_api_key_auth_builder( + request=request, + api_key="Bearer header.payload.signature", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={}, + ) + finally: + for k, v in originals.items(): + setattr(_proxy_server_mod, k, v) + + assert token.team_id == "team-jwt-aliases" + assert token.user_role == (LitellmUserRoles.PROXY_ADMIN if is_proxy_admin else LitellmUserRoles.INTERNAL_USER) + assert token.team_model_aliases == {"fast": "gpt-4o"} + assert token.team_object_permission is not None + assert token.team_object_permission.mcp_servers == ["mcp-a"] + assert token.team_object_permission_id == "op-jwt" + assert token.team_alias == "jwt-aliases" + assert token.team_models == ["gpt-4o"] + assert token.team_max_budget == 40.0 + assert token.team_spend == 4.0 + assert token.team_metadata == {"tier": "gold"} + assert token.team_member == Member(user_id="jwt-user", role="admin") + assert token.team_member_spend == 1.5 + assert token.jwt_claims == {"sub": "jwt-user"} diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 051e6bed4fd..2f6561046b1 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -2205,6 +2205,50 @@ async def test_team_model_add_delete_refresh_team_cache(endpoint_name): assert update_call_kwargs.get("include", {}).get("object_permission") is True +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint_name", ["team_model_add", "team_model_delete"]) +async def test_team_model_add_delete_keep_model_aliases_in_team_cache(endpoint_name, monkeypatch): + """LIT-5858: Prisma only returns `litellm_model_table` when the `update` asks for it, so the refreshed + cache entry lost the team's model aliases and JWT alias requests 403'd until the next DB read.""" + from litellm.proxy._types import TeamModelAddRequest, TeamModelDeleteRequest + from litellm.proxy.auth.team_grants import team_model_aliases + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints.team_endpoints import team_model_add, team_model_delete + + columns = {"team_id": "team-1234", "models": ["gpt-4o", "openai/*"]} + alias_table = {"id": 1, "model_aliases": '{"fast": "gpt-4o"}', "created_by": "admin", "updated_by": "admin"} + + async def update(where, data, include=None): + row = {**columns, "litellm_model_table": alias_table} if (include or {}).get("litellm_model_table") else columns + return SimpleNamespace(team_id="team-1234", model_dump=lambda: row) + + prisma_client = MagicMock() + prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=SimpleNamespace(model_dump=lambda: columns)) + prisma_client.db.litellm_teamtable.update = AsyncMock(side_effect=update) + prisma_client.db.execute_raw = AsyncMock(return_value=None) + cache = UserApiKeyCache() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", None) + + admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") + if endpoint_name == "team_model_add": + await team_model_add( + data=TeamModelAddRequest(team_id="team-1234", models=["team-byok-1"]), + http_request=MagicMock(), + user_api_key_dict=admin, + ) + else: + await team_model_delete( + data=TeamModelDeleteRequest(team_id="team-1234", models=["openai/*"]), + http_request=MagicMock(), + user_api_key_dict=admin, + ) + + cached_team = await cache.async_get_cache(key="team_id:team-1234", model_type=LiteLLM_TeamTableCachedObj) + assert team_model_aliases(cached_team) == {"fast": "gpt-4o"} + + @pytest.mark.asyncio @pytest.mark.parametrize( "endpoint_name", diff --git a/type-discipline-budget.json b/type-discipline-budget.json index e7186dfe186..7db8ed501f8 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 22180 }, "LIT002": { - "limit": 26729 + "limit": 26727 }, "LIT003": { "limit": 261 From 808787659b05ac986a01892938e7387fe88d1a46 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:42:08 +1000 Subject: [PATCH 051/411] fix(azure_ai): stop accepting MAI image params the endpoint cannot honour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two params were advertised for the MAI image models and dropped downstream, so the caller got a 200 that did not match the request, or an opaque provider 400. n: get_supported_openai_params returns ["n", "size"], so n passes validation and is forwarded. The MAI endpoint (/mai/v1/images/generations) has no count field at all — its documented body is model/prompt/width/height, plus image for edits — and ignores both `n` and the native `sampleCount`. Measured against MAI-Image-2.5 and MAI-Image-2.5-Flash: n=2 and n=4 each return HTTP 200 with exactly one image, billed as one, with nothing in the response saying the request was reduced. A caller balancing cost against image count cannot see it. n=1 still passes through; n>1 now raises unless drop_params is set, which is the existing opt-in for silently dropping a param. size: _map_size_param's table offered five sizes, of which one is usable. MAI requires width and height >= 768px and width*height <= 1048576, so 512x512 and 256x256 are under the per-side minimum and 1792x1024 / 1024x1792 are over the pixel budget — all four 400 at the provider with "Model does not support request parameter value supplied: 'width' must be at least 768 pixels." Only 1024x1024 works. The bounds are now checked where the size is mapped, so the error names the constraint instead of arriving from Azure. width/height are deliberately left unchecked: they pass through unmapped, so a future MAI model with different bounds stays reachable without a code change. Verified on a live Azure AI Foundry deployment of MAI-Image-2.5 and MAI-Image-2.5-Flash (2026-08-17). One existing test asserted the 1792x1024 mapping; its size is changed to a size the provider accepts, keeping what it was testing. Co-Authored-By: Claude Opus 5 --- .../image_generation/mai_transformation.py | 49 +++++++- .../test_mai_image_generation.py | 106 +++++++++++++----- 2 files changed, 122 insertions(+), 33 deletions(-) diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 64f81956ad7..3421cc0a4a1 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -21,6 +21,19 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): DEFAULT_WIDTH = 1024 DEFAULT_HEIGHT = 1024 + # The MAI endpoint produces exactly one image per request. Its documented + # body is model/prompt/width/height (plus `image` for edits) — there is no + # count field, and `n` (or the native `sampleCount`) is accepted and + # ignored, so a request for more silently comes back with one. + MAX_IMAGES_PER_REQUEST: Final = 1 + + # Provider-side bounds on the generated image. Both are enforced by the + # MAI endpoint, which 400s with "'width' must be at least 768 pixels." + # Only `size` is checked against them: `width`/`height` pass through + # unmapped, which keeps a future model with different bounds reachable. + MIN_DIMENSION_PX: Final = 768 + MAX_TOTAL_PX: Final = 1024 * 1024 + @staticmethod def get_mai_image_generation_url( api_base: str | None, @@ -146,6 +159,15 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): if k in supported_params: if k == "size" and v: self._map_size_param(v, optional_params) + elif k == "n" and v is not None and v > self.MAX_IMAGES_PER_REQUEST: + if not drop_params: + raise ValueError( + f"n={v} is not supported for model {model}. The Azure AI MAI image " + f"endpoint returns exactly {self.MAX_IMAGES_PER_REQUEST} image per " + "request and ignores any count, so a larger value would silently " + "return fewer images than requested. Send one request per image, or " + "set drop_params=True to drop n." + ) else: optional_params[k] = v elif k in ("width", "height"): @@ -176,13 +198,9 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): if size in size_mapping: width, height = size_mapping[size] - optional_params["width"] = width - optional_params["height"] = height elif "x" in size: try: width, height = map(int, size.lower().split("x")) - optional_params["width"] = width - optional_params["height"] = height except ValueError: raise ValueError(f"Invalid size format: '{size}'. Expected format 'WIDTHxHEIGHT' (e.g., '1024x1024').") else: @@ -191,6 +209,29 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): f"Use a known size (e.g., '1024x1024') or a custom 'WIDTHxHEIGHT' string." ) + self._validate_dimensions(size=size, width=width, height=height) + optional_params["width"] = width + optional_params["height"] = height + + def _validate_dimensions(self, size: str, width: int, height: int) -> None: + """Reject a `size` the MAI endpoint would 400 on. + + Several OpenAI-standard sizes are outside MAI's bounds: 512x512 and + 256x256 fall under the per-side minimum, and 1792x1024 / 1024x1792 + exceed the total pixel budget. Checking here turns an opaque provider + 400 into an error that names the constraint. + """ + if width < self.MIN_DIMENSION_PX or height < self.MIN_DIMENSION_PX: + raise ValueError( + f"Unsupported size value: '{size}'. Azure AI MAI image models require width and " + f"height of at least {self.MIN_DIMENSION_PX} pixels." + ) + if width * height > self.MAX_TOTAL_PX: + raise ValueError( + f"Unsupported size value: '{size}'. Azure AI MAI image models accept at most " + f"{self.MAX_TOTAL_PX} total pixels ({width}x{height} is {width * height})." + ) + def transform_image_generation_response( self, model: str, diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py index 2a44e77ce09..6cbc79fff2b 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py @@ -1,10 +1,8 @@ -import os from unittest.mock import MagicMock import httpx import pytest - import litellm from litellm.llms.azure.azure import AzureChatCompletion from litellm.llms.azure.image_generation import get_azure_image_generation_config @@ -30,9 +28,7 @@ from litellm.utils import get_optional_params_image_gen class TestAzureMAIImageGeneration: def test_is_mai_model(self): assert AzureFoundryMAIImageGenerationConfig.is_mai_model("MAI-Image-2.5") - assert AzureFoundryMAIImageGenerationConfig.is_mai_model( - "azure_ai/MAI-Image-2.5" - ) + assert AzureFoundryMAIImageGenerationConfig.is_mai_model("azure_ai/MAI-Image-2.5") assert AzureFoundryMAIImageGenerationConfig.is_mai_model("MAI-Image-2.5-Flash") assert AzureFoundryMAIImageGenerationConfig.is_mai_model("MAI-Image-2e") assert not AzureFoundryMAIImageGenerationConfig.is_mai_model("flux.2-pro") @@ -62,16 +58,10 @@ class TestAzureMAIImageGeneration: api_base="https://my-resource.services.ai.azure.com", api_version="preview", ) - assert ( - url - == "https://my-resource.services.ai.azure.com/mai/v1/images/generations?api-version=preview" - ) + assert url == "https://my-resource.services.ai.azure.com/mai/v1/images/generations?api-version=preview" def test_get_mai_image_generation_url_preserves_full_path(self): - api = ( - "https://my-resource.services.ai.azure.com/mai/v1/images/generations" - "?api-version=preview" - ) + api = "https://my-resource.services.ai.azure.com/mai/v1/images/generations?api-version=preview" url = AzureFoundryMAIImageGenerationConfig.get_mai_image_generation_url( api_base=api, api_version="preview", @@ -83,10 +73,7 @@ class TestAzureMAIImageGeneration: api_base="https://my-resource.services.ai.azure.com/mai/v1", api_version="preview", ) - assert ( - url - == "https://my-resource.services.ai.azure.com/mai/v1/images/generations?api-version=preview" - ) + assert url == "https://my-resource.services.ai.azure.com/mai/v1/images/generations?api-version=preview" def test_get_azure_ai_image_generation_config_returns_mai(self): config = get_azure_ai_image_generation_config("MAI-Image-2.5") @@ -124,13 +111,13 @@ class TestAzureMAIImageGeneration: config = AzureFoundryMAIImageGenerationConfig() optional_params = get_optional_params_image_gen( model="MAI-Image-2.5", - size="1792x1024", + size="1024x1024", n=1, custom_llm_provider="azure_ai", provider_config=config, drop_params=True, ) - assert optional_params["width"] == 1792 + assert optional_params["width"] == 1024 assert optional_params["height"] == 1024 assert "size" not in optional_params @@ -147,10 +134,7 @@ class TestAzureMAIImageGeneration: assert "api-version=preview" in url def test_mai_json_body_keeps_model(self): - api = ( - "https://my-resource.services.ai.azure.com/mai/v1/images/generations" - "?api-version=preview" - ) + api = "https://my-resource.services.ai.azure.com/mai/v1/images/generations?api-version=preview" data = { "model": "MAI-Image-2.5", "prompt": "A photograph of a red fox", @@ -214,6 +198,74 @@ class TestAzureMAIImageGeneration: drop_params=True, ) + @pytest.mark.parametrize("size", ["512x512", "256x256", "700x1400"]) + def test_map_openai_params_size_below_minimum_dimension_raises(self, size): + """MAI requires >= 768px per side; the OpenAI size table offered smaller ones.""" + config = AzureFoundryMAIImageGenerationConfig() + with pytest.raises(ValueError, match="at least 768 pixels"): + config.map_openai_params( + non_default_params={"size": size}, + optional_params={}, + model="MAI-Image-2.5", + drop_params=True, + ) + + @pytest.mark.parametrize("size", ["1792x1024", "1024x1792"]) + def test_map_openai_params_size_over_total_pixel_budget_raises(self, size): + """MAI caps total pixels at 1024*1024, so both landscape/portrait sizes 400 upstream.""" + config = AzureFoundryMAIImageGenerationConfig() + with pytest.raises(ValueError, match="at most 1048576 total pixels"): + config.map_openai_params( + non_default_params={"size": size}, + optional_params={}, + model="MAI-Image-2.5", + drop_params=True, + ) + + def test_map_openai_params_explicit_width_height_not_range_checked(self): + """width/height pass through unmapped, so a future model's bounds stay reachable.""" + config = AzureFoundryMAIImageGenerationConfig() + optional_params = config.map_openai_params( + non_default_params={"width": 1792, "height": 1024}, + optional_params={}, + model="MAI-Image-2.5", + drop_params=True, + ) + assert optional_params["width"] == 1792 + assert optional_params["height"] == 1024 + + @pytest.mark.parametrize("n", [2, 4]) + def test_map_openai_params_multi_image_n_raises(self, n): + """The MAI endpoint returns one image and ignores any count, so n>1 must not pass silently.""" + config = AzureFoundryMAIImageGenerationConfig() + with pytest.raises(ValueError, match="returns exactly 1 image per request"): + config.map_openai_params( + non_default_params={"n": n}, + optional_params={}, + model="MAI-Image-2.5", + drop_params=False, + ) + + def test_map_openai_params_multi_image_n_dropped_with_drop_params(self): + config = AzureFoundryMAIImageGenerationConfig() + optional_params = config.map_openai_params( + non_default_params={"n": 4}, + optional_params={}, + model="MAI-Image-2.5", + drop_params=True, + ) + assert "n" not in optional_params + + def test_map_openai_params_single_image_n_still_passes_through(self): + config = AzureFoundryMAIImageGenerationConfig() + optional_params = config.map_openai_params( + non_default_params={"n": 1}, + optional_params={}, + model="MAI-Image-2.5", + drop_params=False, + ) + assert optional_params["n"] == 1 + def test_map_openai_params_unsupported_param_raises(self): config = AzureFoundryMAIImageGenerationConfig() with pytest.raises(ValueError, match="Parameter quality is not supported"): @@ -363,16 +415,12 @@ class TestAzureMAIImageGeneration: litellm.model_cost = litellm.get_model_cost_map(url="") model = "azure_ai/MAI-Image-2.5" model_info = litellm.get_model_info(model=model, custom_llm_provider="azure_ai") - image_response = ImageResponse( - data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] - ) + image_response = ImageResponse(data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")]) cost = azure_ai_image_cost_calculator( model=model, image_response=image_response, ) - assert ( - cost == len(image_response.data or []) * model_info["output_cost_per_image"] - ) + assert cost == len(image_response.data or []) * model_info["output_cost_per_image"] assert cost > 0 From 1d8650413d4237f9dfce6aec838035618a858d8b Mon Sep 17 00:00:00 2001 From: jesus Date: Mon, 7 Sep 2026 11:51:35 +0000 Subject: [PATCH 052/411] fix(responses): keep background polling alive after the client disconnects Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../response_polling/background_streaming.py | 17 +++++- .../test_response_polling_handler.py | 4 +- .../proxy/response_polling/__init__.py | 0 .../test_background_streaming.py | 52 +++++++++++++++++++ 4 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/proxy/response_polling/__init__.py create mode 100644 tests/test_litellm/proxy/response_polling/test_background_streaming.py diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 0fd242f2bc1..fac45d4391c 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Final, TypeAlias from fastapi import Request, Response from fastapi.responses import StreamingResponse +from starlette.types import Message from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -74,6 +75,20 @@ class _StreamEventParser: parse: Callable[[str], _StreamEvent] = staticmethod(json.loads) +async def _never_receive() -> Message: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + +def detach_request_from_client(request: Request) -> Request: + """Same scope (headers, parsed body, auth) but a receive() that never yields http.disconnect. + + The polling client closes its connection right after getting the polling id, so the + upstream call must not be cancelled by the client-disconnect guards. + """ + return Request(request.scope, _never_receive) + + async def background_streaming_task( polling_id: str, data: dict[str, object], @@ -123,7 +138,7 @@ async def background_streaming_task( # Pre-call checks (rate limits, guardrails, budget) were already run # before polling ID creation, so skip them here to avoid double-counting. response: Final[StreamingResponse] = await processor.base_process_llm_request( - request=request, + request=detach_request_from_client(request), fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, route_type="aresponses", diff --git a/tests/proxy_unit_tests/test_response_polling_handler.py b/tests/proxy_unit_tests/test_response_polling_handler.py index 772d3622745..7a4e0ca89c0 100644 --- a/tests/proxy_unit_tests/test_response_polling_handler.py +++ b/tests/proxy_unit_tests/test_response_polling_handler.py @@ -20,7 +20,7 @@ from typing import Any, Dict, Optional from unittest.mock import AsyncMock, Mock, patch import pytest - +from fastapi import Request from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler @@ -1414,7 +1414,7 @@ def _make_background_streaming_kwargs( polling_id=polling_id, data={"model": "gpt-4o", "stream": False, "background": True}, polling_handler=polling_handler, - request=Mock(), + request=Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}), fastapi_response=Mock(), user_api_key_dict=Mock(), general_settings={}, diff --git a/tests/test_litellm/proxy/response_polling/__init__.py b/tests/test_litellm/proxy/response_polling/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/response_polling/test_background_streaming.py b/tests/test_litellm/proxy/response_polling/test_background_streaming.py new file mode 100644 index 00000000000..98d914b1848 --- /dev/null +++ b/tests/test_litellm/proxy/response_polling/test_background_streaming.py @@ -0,0 +1,52 @@ +import asyncio +from collections.abc import AsyncGenerator +from typing import Final + +import pytest +from fastapi import Request +from fastapi.responses import JSONResponse, StreamingResponse +from starlette.types import Message + +from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED +from litellm.proxy.common_request_processing import create_response +from litellm.proxy.response_polling.background_streaming import detach_request_from_client + + +def _request_whose_client_already_left() -> Request: + async def receive() -> Message: + return {"type": "http.disconnect"} + + scope: Final = { + "type": "http", + "method": "POST", + "path": "/v1/responses", + "headers": [(b"x-litellm-call-id", b"call-123")], + "query_string": b"", + } + return Request(scope, receive) + + +async def _slow_first_chunk() -> AsyncGenerator[str, None]: + await asyncio.sleep(0.05) + yield 'data: {"type": "response.created"}\n\n' + + +@pytest.mark.asyncio +async def test_detached_request_survives_client_disconnect_before_first_chunk(): + original: Final = _request_whose_client_already_left() + + cancelled: Final = await create_response(_slow_first_chunk(), "text/event-stream", {}, request=original) + assert isinstance(cancelled, JSONResponse) + assert cancelled.status_code == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + + detached: Final = detach_request_from_client(original) + kept_alive: Final = await create_response(_slow_first_chunk(), "text/event-stream", {}, request=detached) + assert isinstance(kept_alive, StreamingResponse) + assert kept_alive.status_code == 200 + + +def test_detached_request_keeps_scope(): + original: Final = _request_whose_client_already_left() + detached: Final = detach_request_from_client(original) + assert detached.headers["x-litellm-call-id"] == "call-123" + assert detached.scope is original.scope From 9c8caae1e4a793d17493f4de6caebf2cfa1ad651 Mon Sep 17 00:00:00 2001 From: jesus Date: Mon, 7 Sep 2026 11:53:50 +0000 Subject: [PATCH 053/411] ci: add tests/test_litellm/proxy/response_polling to the proxy-endpoints shard Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-unit.yml | 1 + Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 33245ec5b5f..761960145eb 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -155,6 +155,7 @@ jobs: tests/test_litellm/proxy/vector_store_files_endpoints tests/test_litellm/proxy/video_endpoints tests/test_litellm/proxy/response_api_endpoints + tests/test_litellm/proxy/response_polling tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/ocr_endpoints tests/test_litellm/proxy/vector_store_endpoints diff --git a/Makefile b/Makefile index e17fdba3c85..b14aaaf896f 100644 --- a/Makefile +++ b/Makefile @@ -298,7 +298,7 @@ test-unit-proxy-core: install-test-deps $(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20 test-unit-proxy-misc: install-test-deps - $(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/response_polling tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 test-unit-integrations: install-test-deps $(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20 From e589a0ed82023d6fa550e6ddfa005c5ebbf4951e Mon Sep 17 00:00:00 2001 From: jesus Date: Mon, 7 Sep 2026 12:07:31 +0000 Subject: [PATCH 054/411] test(responses): cover the polling disconnect regression through background_streaming_task Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-unit.yml | 1 - Makefile | 2 +- .../test_response_polling_handler.py | 58 +++++++++++++++++++ .../proxy/response_polling/__init__.py | 0 .../test_background_streaming.py | 52 ----------------- 5 files changed, 59 insertions(+), 54 deletions(-) delete mode 100644 tests/test_litellm/proxy/response_polling/__init__.py delete mode 100644 tests/test_litellm/proxy/response_polling/test_background_streaming.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 761960145eb..33245ec5b5f 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -155,7 +155,6 @@ jobs: tests/test_litellm/proxy/vector_store_files_endpoints tests/test_litellm/proxy/video_endpoints tests/test_litellm/proxy/response_api_endpoints - tests/test_litellm/proxy/response_polling tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/ocr_endpoints tests/test_litellm/proxy/vector_store_endpoints diff --git a/Makefile b/Makefile index b14aaaf896f..e17fdba3c85 100644 --- a/Makefile +++ b/Makefile @@ -298,7 +298,7 @@ test-unit-proxy-core: install-test-deps $(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20 test-unit-proxy-misc: install-test-deps - $(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/response_polling tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 test-unit-integrations: install-test-deps $(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20 diff --git a/tests/proxy_unit_tests/test_response_polling_handler.py b/tests/proxy_unit_tests/test_response_polling_handler.py index 7a4e0ca89c0..f5335508ce4 100644 --- a/tests/proxy_unit_tests/test_response_polling_handler.py +++ b/tests/proxy_unit_tests/test_response_polling_handler.py @@ -14,6 +14,7 @@ These tests ensure the polling handler correctly manages response state following the OpenAI Response API format. """ +import asyncio import json from datetime import datetime, timezone from typing import Any, Dict, Optional @@ -1663,6 +1664,63 @@ class TestBackgroundStreamingTerminalEvents: final_call = handler.update_state.call_args_list[-1] assert final_call.kwargs["status"] == "completed" + @pytest.mark.asyncio + async def test_polling_client_disconnect_does_not_cancel_upstream_call(self): + """The polling client hangs up right after getting its polling id. The detached task + must still stream the upstream response through the client-disconnect guards.""" + from litellm.proxy.common_request_processing import create_response + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + async def client_already_left(): + return {"type": "http.disconnect"} + + async def slow_upstream_stream(): + await asyncio.sleep(0.05) + for event in ( + {"type": "response.in_progress"}, + { + "type": "response.completed", + "response": { + "id": "resp_123", + "status": "completed", + "usage": {"input_tokens": 13, "output_tokens": 10}, + "model": "gpt-4o", + "output": [{"id": "item_1", "type": "message"}], + }, + }, + ): + yield f"data: {json.dumps(event)}\n\n" + + async def upstream_call_behind_disconnect_guard(**kwargs): + return await create_response( + slow_upstream_stream(), "text/event-stream", {}, request=kwargs["request"] + ) + + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_7", handler) + kwargs["request"] = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/responses", + "headers": [(b"x-litellm-call-id", b"call-123")], + "query_string": b"", + }, + client_already_left, + ) + + with patch( + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = upstream_call_behind_disconnect_guard + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "completed" + assert final_call.kwargs["usage"] == {"input_tokens": 13, "output_tokens": 10} + class TestEdgeCases: """Test edge cases and error scenarios""" diff --git a/tests/test_litellm/proxy/response_polling/__init__.py b/tests/test_litellm/proxy/response_polling/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/test_litellm/proxy/response_polling/test_background_streaming.py b/tests/test_litellm/proxy/response_polling/test_background_streaming.py deleted file mode 100644 index 98d914b1848..00000000000 --- a/tests/test_litellm/proxy/response_polling/test_background_streaming.py +++ /dev/null @@ -1,52 +0,0 @@ -import asyncio -from collections.abc import AsyncGenerator -from typing import Final - -import pytest -from fastapi import Request -from fastapi.responses import JSONResponse, StreamingResponse -from starlette.types import Message - -from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED -from litellm.proxy.common_request_processing import create_response -from litellm.proxy.response_polling.background_streaming import detach_request_from_client - - -def _request_whose_client_already_left() -> Request: - async def receive() -> Message: - return {"type": "http.disconnect"} - - scope: Final = { - "type": "http", - "method": "POST", - "path": "/v1/responses", - "headers": [(b"x-litellm-call-id", b"call-123")], - "query_string": b"", - } - return Request(scope, receive) - - -async def _slow_first_chunk() -> AsyncGenerator[str, None]: - await asyncio.sleep(0.05) - yield 'data: {"type": "response.created"}\n\n' - - -@pytest.mark.asyncio -async def test_detached_request_survives_client_disconnect_before_first_chunk(): - original: Final = _request_whose_client_already_left() - - cancelled: Final = await create_response(_slow_first_chunk(), "text/event-stream", {}, request=original) - assert isinstance(cancelled, JSONResponse) - assert cancelled.status_code == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED - - detached: Final = detach_request_from_client(original) - kept_alive: Final = await create_response(_slow_first_chunk(), "text/event-stream", {}, request=detached) - assert isinstance(kept_alive, StreamingResponse) - assert kept_alive.status_code == 200 - - -def test_detached_request_keeps_scope(): - original: Final = _request_whose_client_already_left() - detached: Final = detach_request_from_client(original) - assert detached.headers["x-litellm-call-id"] == "call-123" - assert detached.scope is original.scope From 8576cf74c2ef4eaa15f48eaaf99447f9d6fb394d Mon Sep 17 00:00:00 2001 From: Oliver Jensen Date: Mon, 7 Sep 2026 14:09:37 +0200 Subject: [PATCH 055/411] feat(auth): add disable_env_credential_login setting with admin ui warning Env-credential login (UI_USERNAME/UI_PASSWORD, or the master key when UI_PASSWORD is unset) is always live today. This adds a general_settings flag to turn that login path off once real admin accounts exist, and a warning banner shown to any admin while it remains enabled. The banner flag is served through /health/readiness/details and stays quiet when disable_password_login_when_sso_enabled already makes the env path unreachable. --- litellm/proxy/_types.py | 12 ++ litellm/proxy/auth/login_utils.py | 32 +++- .../health_endpoints/_health_endpoints.py | 20 +++ .../proxy/auth/test_login_utils.py | 154 ++++++++++++++++++ .../health_endpoints/test_health_endpoints.py | 27 +++ .../useHealthReadinessDetails.ts | 1 + .../src/app/(dashboard)/layout.test.tsx | 4 + .../src/app/(dashboard)/layout.tsx | 3 + .../EnvCredentialLoginWarningBanner.test.tsx | 76 +++++++++ .../EnvCredentialLoginWarningBanner.tsx | 35 ++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 11 files changed, 364 insertions(+), 5 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.test.tsx create mode 100644 ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.tsx diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c28ac8848ba..c30a3c5bea8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2817,6 +2817,18 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): "UI username/password login. Default is False." ), ) + disable_env_credential_login: bool | None = Field( + None, + description=( + "If True, disables signing in to the Admin UI with the environment credentials: " + "UI_USERNAME/UI_PASSWORD, or the master key when UI_PASSWORD is unset (that fallback " + "means env-credential login is always live by default). Database users with passwords " + "are unaffected. LOCKOUT RISK: create at least one proxy admin user with a password " + "before enabling, or nobody can sign in to the UI. A locked-out admin can still " + "administer the proxy over the API with the master key, and can unset this setting " + "and restart the proxy to restore env-credential login. Default is False." + ), + ) disable_budget_reservation: bool | None = Field( None, description=( diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 8d4f6f81363..6ca7ea5074d 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -85,6 +85,29 @@ def get_ui_credentials(master_key: str | None) -> tuple[str, str]: return ui_username, ui_password +def _matches_env_credentials(username: str, password: str, master_key: str | None) -> bool: + ui_username, ui_password = get_ui_credentials(master_key) + return secrets.compare_digest(username.encode("utf-8"), ui_username.encode("utf-8")) and secrets.compare_digest( + password.encode("utf-8"), ui_password.encode("utf-8") + ) + + +def is_env_credential_login_enabled(general_settings: Mapping[str, object]) -> bool: + """Whether a login with UI_USERNAME/UI_PASSWORD (or the master-key fallback) can succeed. + + Two settings can turn it off: `disable_env_credential_login` unconditionally, and + `disable_password_login_when_sso_enabled` as a side effect, since its gate rejects + every username/password login before the env comparison runs. Feeds both the + `authenticate_user` gate and the Admin UI warning banner, so the banner never nags + about a login path that is already unreachable. + """ + if general_settings.get("disable_env_credential_login") is True: + return False + if general_settings.get("disable_password_login_when_sso_enabled") is True and is_sso_provider_fully_configured(): + return False + return True + + class LoginResult: """Result object containing authentication data from login.""" @@ -129,7 +152,8 @@ async def authenticate_user( master_key: Master key for the proxy (required) prisma_client: Prisma database client (optional) general_settings: Proxy general_settings, checked for - `disable_password_login_when_sso_enabled` + `disable_password_login_when_sso_enabled` and + `disable_env_credential_login` Returns: LoginResult: Object containing authentication data @@ -170,8 +194,6 @@ async def authenticate_user( code=500, ) - ui_username, ui_password = get_ui_credentials(master_key) - # Check if we can find the `username` in the db. On the UI, users can enter username=their email _user_row: LiteLLM_UserTable | None = None user_role: ( @@ -197,8 +219,8 @@ async def authenticate_user( - Login with UI_USERNAME and UI_PASSWORD - Login with Invite Link `user_email` and `password` combination """ - if secrets.compare_digest(username.encode("utf-8"), ui_username.encode("utf-8")) and secrets.compare_digest( - password.encode("utf-8"), ui_password.encode("utf-8") + if general_settings.get("disable_env_credential_login") is not True and _matches_env_credentials( + username, password, master_key ): # Non SSO -> If user is using UI_USERNAME and UI_PASSWORD they are Proxy admin user_role = LitellmUserRoles.PROXY_ADMIN diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 1785a2f0992..e5e4f89234b 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1582,6 +1582,23 @@ async def _show_no_redis_warning() -> bool: return await count_live_proxy_workers(prisma_client) != 1 +def _show_env_credential_login_warning() -> bool: + """ + Whether the UI should warn admins that env-credential login is still enabled. + + UI_USERNAME/UI_PASSWORD (or the master key, when UI_PASSWORD is unset) grant + proxy-admin access with a shared static secret: no per-person identity, no + audit trail, no password policy, and it stays valid until the env var or + master key rotates. That is fine for first-time setup, so it is on by + default, but once real admin accounts exist it should be turned off with + `general_settings.disable_env_credential_login`. + """ + from litellm.proxy.auth.login_utils import is_env_credential_login_enabled + from litellm.proxy.proxy_server import general_settings + + return is_env_credential_login_enabled(general_settings) + + async def _get_health_readiness_details( response: Response | None = None, ) -> dict[str, Any]: @@ -1623,6 +1640,7 @@ async def _get_health_readiness_details( log_level_name: Final = logging.getLevelName(verbose_logger.getEffectiveLevel()) is_detailed_debug: Final = verbose_logger.isEnabledFor(logging.DEBUG) show_no_redis_warning: Final = await _show_no_redis_warning() + show_env_credential_login_warning: Final = _show_env_credential_login_warning() # check DB if prisma_client is not None: # if db passed in, check if it's connected @@ -1650,6 +1668,7 @@ async def _get_health_readiness_details( "log_level": log_level_name, "is_detailed_debug": is_detailed_debug, "show_no_redis_warning": show_no_redis_warning, + "show_env_credential_login_warning": show_env_credential_login_warning, } else: return { @@ -1662,6 +1681,7 @@ async def _get_health_readiness_details( "log_level": log_level_name, "is_detailed_debug": is_detailed_debug, "show_no_redis_warning": show_no_redis_warning, + "show_env_credential_login_warning": show_env_credential_login_warning, } except Exception as e: raise HTTPException(status_code=503, detail=f"Service Unhealthy ({e})") diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 8d93d801bfd..72cc6e04a12 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -23,6 +23,7 @@ from litellm.proxy.auth.login_utils import ( LoginResult, authenticate_user, get_ui_credentials, + is_env_credential_login_enabled, ) @@ -799,3 +800,156 @@ class TestDisablePasswordLoginWhenSSOEnabled: assert isinstance(result, LoginResult) assert result.user_id == LITELLM_PROXY_ADMIN_NAME + + +class TestDisableEnvCredentialLogin: + """`disable_env_credential_login` must reject a login with the env + credentials (UI_USERNAME/UI_PASSWORD, or the master-key fallback when + UI_PASSWORD is unset) while leaving database-user password logins + untouched, so admins with real accounts keep a way in.""" + + @pytest.mark.asyncio + async def test_rejects_correct_env_credentials_when_disabled(self): + master_key = "sk-1234" + ui_username = "admin" + ui_password = "env-only-password" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict(os.environ, {"UI_USERNAME": ui_username, "UI_PASSWORD": ui_password}): + with pytest.raises(ProxyException) as exc_info: + await authenticate_user( + username=ui_username, + password=ui_password, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_env_credential_login": True}, + ) + + assert exc_info.value.type == ProxyErrorTypes.auth_error + assert exc_info.value.code == "401" + + @pytest.mark.asyncio + async def test_rejects_master_key_fallback_when_disabled(self): + """With UI_PASSWORD unset, the master key IS the env password, so the + setting must reject it too or it protects nothing by default.""" + master_key = "sk-1234" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict(os.environ, {"UI_USERNAME": "admin"}, clear=True): + with pytest.raises(ProxyException) as exc_info: + await authenticate_user( + username="admin", + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_env_credential_login": True}, + ) + + assert exc_info.value.code == "401" + + @pytest.mark.asyncio + async def test_db_user_login_still_works_when_disabled(self): + master_key = "sk-1234" + user_email = "admin@example.com" + password = "Str0ng!Passw0rd" + + mock_user = LiteLLM_UserTable( + user_id="db-admin-1", + user_email=user_email, + password=hash_token(token=password), + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=mock_user) + + with patch.dict( + os.environ, + { + "UI_USERNAME": "admin", + "UI_PASSWORD": "env-password", + "DATABASE_URL": "postgresql://test:test@localhost/test", + }, + clear=True, + ): + with ExitStack() as stack: + stack.enter_context( + patch( # test-quality-ok: internal orchestration, no HTTP boundary; matches pre-existing tests + "litellm.proxy.auth.login_utils.generate_key_helper_fn", + new_callable=AsyncMock, + return_value={"token": "db-user-token"}, + ) + ) + result = await authenticate_user( + username=user_email, + password=password, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_env_credential_login": True}, + ) + + assert isinstance(result, LoginResult) + assert result.user_id == "db-admin-1" + assert result.user_role == LitellmUserRoles.PROXY_ADMIN + + @pytest.mark.asyncio + async def test_env_login_still_works_when_setting_absent(self): + """Env-credential login is the bootstrap path on a fresh install and + must stay on by default.""" + master_key = "sk-1234" + ui_username = "admin" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict( + os.environ, + { + "UI_USERNAME": ui_username, + "UI_PASSWORD": master_key, + "DATABASE_URL": "postgresql://test:test@localhost/test", + }, + clear=True, + ): + with ExitStack() as stack: + _patch_successful_admin_login_deps(stack) + result = await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={}, + ) + + assert isinstance(result, LoginResult) + assert result.user_id == LITELLM_PROXY_ADMIN_NAME + + +class TestIsEnvCredentialLoginEnabled: + """Drives the Admin UI warning banner: it must be True exactly when a + login with the env credentials could actually succeed.""" + + def test_enabled_by_default(self): + assert is_env_credential_login_enabled({}) is True + + def test_disabled_by_dedicated_setting(self): + assert is_env_credential_login_enabled({"disable_env_credential_login": True}) is False + + def test_explicit_false_keeps_it_enabled(self): + assert is_env_credential_login_enabled({"disable_env_credential_login": False}) is True + + def test_disabled_when_sso_gate_blocks_all_password_logins(self): + """`disable_password_login_when_sso_enabled` with SSO configured + rejects every username/password login before the env comparison runs, + so the banner must not nag about an already-unreachable path.""" + with ExitStack() as stack: + _patch_sso_configured(stack, configured=True) + assert is_env_credential_login_enabled({"disable_password_login_when_sso_enabled": True}) is False + + def test_enabled_when_sso_gate_is_set_but_sso_not_configured(self): + with ExitStack() as stack: + _patch_sso_configured(stack, configured=False) + assert is_env_credential_login_enabled({"disable_password_login_when_sso_enabled": True}) is True diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index e9e58347337..b06d87ac67e 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -1301,6 +1301,33 @@ def test_health_readiness_details_returns_diagnostic_fields(monkeypatch): assert "cache" in response_data +@pytest.mark.parametrize( + "general_settings, expected_warning", + [ + ({}, True), + ({"disable_env_credential_login": True}, False), + ], +) +def test_health_readiness_details_reports_env_credential_login_warning(monkeypatch, general_settings, expected_warning): + """ + The Admin UI banner is driven by this flag: it must be True while + env-credential login is possible and False once + `disable_env_credential_login` turns that login path off. + """ + app = FastAPI() + app.include_router(_health_endpoints_module.router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + client = TestClient(app) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + + response = client.get("/health/readiness/details") + + assert response.status_code == 200, response.text + assert response.json()["show_env_credential_login_warning"] is expected_warning + + def test_health_readiness_allows_explicit_legacy_public_details(monkeypatch): """ Operators can explicitly preserve the legacy public readiness payload. diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails.ts index 307fa9e1691..44d9092df34 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails.ts @@ -14,6 +14,7 @@ export interface HealthReadinessDetailsResponse { log_level?: string; is_detailed_debug?: boolean; show_no_redis_warning?: boolean; + show_env_credential_login_warning?: boolean; } const fetchHealthReadinessDetails = async (accessToken: string): Promise => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx index 7f1f4cc4bd5..3fe34610260 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx @@ -29,6 +29,10 @@ vi.mock("@/components/NoRedisWarningBanner", () => ({ NoRedisWarningBanner: () => null, })); +vi.mock("@/components/EnvCredentialLoginWarningBanner", () => ({ + EnvCredentialLoginWarningBanner: () => null, +})); + vi.mock("@/components/LicenseExpiryBanner", () => ({ LicenseExpiryBanner: () => null, })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index 98f2a36d6f3..fa6df7f176a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -10,6 +10,7 @@ import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider"; import { useRouter, useSearchParams } from "next/navigation"; import { DebugWarningBanner } from "@/components/DebugWarningBanner"; import { NoRedisWarningBanner } from "@/components/NoRedisWarningBanner"; +import { EnvCredentialLoginWarningBanner } from "@/components/EnvCredentialLoginWarningBanner"; import { LicenseExpiryBanner } from "@/components/LicenseExpiryBanner"; import { UserBanner } from "@/components/UserBanner"; import { uiHref } from "@/utils/uiHref"; @@ -113,6 +114,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { +
@@ -132,6 +134,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { +
{children}
diff --git a/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.test.tsx b/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.test.tsx new file mode 100644 index 00000000000..535694f14da --- /dev/null +++ b/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.test.tsx @@ -0,0 +1,76 @@ +import { renderWithProviders, screen } from "../../tests/test-utils"; +import { vi } from "vitest"; +import { EnvCredentialLoginWarningBanner } from "./EnvCredentialLoginWarningBanner"; +import type { HealthReadinessDetailsResponse } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; +import type { UseQueryResult } from "@tanstack/react-query"; + +vi.mock("@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails", () => ({ + useHealthReadinessDetails: vi.fn(), +})); +vi.mock("@/contexts/AuthContext", () => ({ + useAuth: vi.fn(), +})); + +import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; +import { useAuth } from "@/contexts/AuthContext"; + +const mockDetails = (data: Partial | undefined) => { + vi.mocked(useHealthReadinessDetails).mockReturnValue({ data } as UseQueryResult); +}; + +const mockRole = (userRole: string) => { + vi.mocked(useAuth).mockReturnValue({ userRole } as ReturnType); +}; + +describe("EnvCredentialLoginWarningBanner", () => { + it("should warn an admin when env-credential login is enabled", () => { + mockRole("Admin"); + mockDetails({ status: "healthy", show_env_credential_login_warning: true }); + renderWithProviders(); + expect(screen.getByRole("alert")).toBeInTheDocument(); + expect(screen.getByText("Environment-credential login is enabled")).toBeInTheDocument(); + }); + + it("should tell the admin to create a regular admin account before disabling", () => { + mockRole("Admin"); + mockDetails({ status: "healthy", show_env_credential_login_warning: true }); + renderWithProviders(); + expect(screen.getByText(/First create a regular admin account/i)).toBeInTheDocument(); + expect(screen.getByText("general_settings.disable_env_credential_login: true")).toBeInTheDocument(); + }); + + it("should warn an admin viewer too", () => { + mockRole("Admin Viewer"); + mockDetails({ status: "healthy", show_env_credential_login_warning: true }); + renderWithProviders(); + expect(screen.getByRole("alert")).toBeInTheDocument(); + }); + + it("should render nothing for a non-admin even when the proxy reports the warning", () => { + mockRole("Internal User"); + mockDetails({ status: "healthy", show_env_credential_login_warning: true }); + const { container } = renderWithProviders(); + expect(container).toBeEmptyDOMElement(); + }); + + it("should render nothing when env-credential login is disabled", () => { + mockRole("Admin"); + mockDetails({ status: "healthy", show_env_credential_login_warning: false }); + const { container } = renderWithProviders(); + expect(container).toBeEmptyDOMElement(); + }); + + it("should render nothing when readiness details are unavailable", () => { + mockRole("Admin"); + mockDetails(undefined); + const { container } = renderWithProviders(); + expect(container).toBeEmptyDOMElement(); + }); + + it("should pass the access token to the readiness hook", () => { + mockRole("Admin"); + mockDetails(undefined); + renderWithProviders(); + expect(useHealthReadinessDetails).toHaveBeenCalledWith("my-token"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.tsx b/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.tsx new file mode 100644 index 00000000000..3a9d50011f1 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EnvCredentialLoginWarningBanner.tsx @@ -0,0 +1,35 @@ +"use client"; + +import React from "react"; +import { TriangleAlert } from "lucide-react"; +import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; +import { useAuth } from "@/contexts/AuthContext"; +import { isAdminRole } from "@/utils/roles"; + +export const EnvCredentialLoginWarningBanner: React.FC<{ accessToken: string | null }> = ({ accessToken }) => { + const { userRole } = useAuth(); + const { data: healthData } = useHealthReadinessDetails(accessToken); + + if (!isAdminRole(userRole) || !healthData?.show_env_credential_login_warning) { + return null; + } + + return ( +
+
+ ); +}; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b1534c19670..dae0af3d3b1 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25778,6 +25778,11 @@ export interface components { * @description If True, disables the optimistic per-request budget reservation introduced in v1.84.0. WARNING: This weakens hard budget enforcement. Without the reservation, a burst of concurrent requests from a single key can each pass the read-time spend check before any of them is charged, allowing a configured budget to be exceeded under high concurrency. Budgets are still evaluated on every request at read time, so an already-exhausted budget is still rejected. Enable only if your deployment is experiencing phantom BudgetExceededError responses caused by leaked reservations (see GitHub issue #27639). A proxy-level WARNING is logged on every request while this flag is active as a reminder that hard enforcement is relaxed. */ disable_budget_reservation?: boolean | null; + /** + * Disable Env Credential Login + * @description If True, disables signing in to the Admin UI with the environment credentials: UI_USERNAME/UI_PASSWORD, or the master key when UI_PASSWORD is unset (that fallback means env-credential login is always live by default). Database users with passwords are unaffected. LOCKOUT RISK: create at least one proxy admin user with a password before enabling, or nobody can sign in to the UI. A locked-out admin can still administer the proxy over the API with the master key, and can unset this setting and restart the proxy to restore env-credential login. Default is False. + */ + disable_env_credential_login?: boolean | null; /** * Disable Password Login When Sso Enabled * @description If True and SSO is configured (MICROSOFT_CLIENT_ID, GOOGLE_CLIENT_ID, GENERIC_CLIENT_ID, or SAML_IDP_METADATA_URL/XML), disables username/password login on /login, /v2/login, and /v3/login so SSO is the only way to reach the Admin UI. An admin locked out of the UI can still administer the proxy over the API with the master key; unset this setting and restart the proxy to restore UI username/password login. Default is False. From 0d27ec952aa6b1df6697d952b90e9ea4569c06e5 Mon Sep 17 00:00:00 2001 From: jesus Date: Mon, 7 Sep 2026 12:14:36 +0000 Subject: [PATCH 056/411] test(responses): annotate the processor patch for the test quality gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/proxy_unit_tests/test_response_polling_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/proxy_unit_tests/test_response_polling_handler.py b/tests/proxy_unit_tests/test_response_polling_handler.py index f5335508ce4..467c1332325 100644 --- a/tests/proxy_unit_tests/test_response_polling_handler.py +++ b/tests/proxy_unit_tests/test_response_polling_handler.py @@ -1711,7 +1711,7 @@ class TestBackgroundStreamingTerminalEvents: client_already_left, ) - with patch( + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" ) as MockProcessor: MockProcessor.return_value.base_process_llm_request = upstream_call_behind_disconnect_guard From 15c9fca53fc04c5ab05ebba0a8f047ef77394abd Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 7 Sep 2026 13:17:30 +0000 Subject: [PATCH 057/411] fix(pricing): add xai grok-imagine-video entries, computer-use-preview deprecation, gemini live 2.5 native audio card values Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 94 +++++++++++++++++-- model_prices_and_context_window.json | 94 +++++++++++++++++-- 2 files changed, 174 insertions(+), 14 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c558a953032..33e9ce94455 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14336,6 +14336,7 @@ "supports_tool_choice": true }, "computer-use-preview": { + "deprecation_date": "2026-07-23", "input_cost_per_token": 3e-06, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -23829,9 +23830,9 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -23854,12 +23855,12 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_pdf_input": false, + "supports_prompt_caching": false, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_url_context": true, + "supports_url_context": false, "supports_vision": true, "supports_web_search": true, "search_context_cost_per_query": { @@ -59479,6 +59480,85 @@ "image" ] }, + "xai/grok-imagine-video": { + "input_cost_per_image": 0.002, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_480p": 0.05, + "source": "https://docs.x.ai/docs/models/grok-imagine-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-2026-05-30": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-preview": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, "low/1024-x-1024/grok-imagine-image-2.0": { "input_cost_per_image": 0.04, "litellm_provider": "xai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c558a953032..33e9ce94455 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14336,6 +14336,7 @@ "supports_tool_choice": true }, "computer-use-preview": { + "deprecation_date": "2026-07-23", "input_cost_per_token": 3e-06, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -23829,9 +23830,9 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -23854,12 +23855,12 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_pdf_input": false, + "supports_prompt_caching": false, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_url_context": true, + "supports_url_context": false, "supports_vision": true, "supports_web_search": true, "search_context_cost_per_query": { @@ -59479,6 +59480,85 @@ "image" ] }, + "xai/grok-imagine-video": { + "input_cost_per_image": 0.002, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_480p": 0.05, + "source": "https://docs.x.ai/docs/models/grok-imagine-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-2026-05-30": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-preview": { + "input_cost_per_image": 0.01, + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, "low/1024-x-1024/grok-imagine-image-2.0": { "input_cost_per_image": 0.04, "litellm_provider": "xai", From 68d7a2d98fadd8f15eec9d5b0005e79de33e98bb Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 7 Sep 2026 13:29:29 +0000 Subject: [PATCH 058/411] fix(pricing): drop supported_endpoints from xai grok-imagine-video entries to match video model convention Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 12 ------------ model_prices_and_context_window.json | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 33e9ce94455..537db678f60 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -59487,9 +59487,6 @@ "output_cost_per_second": 0.05, "output_cost_per_second_480p": 0.05, "source": "https://docs.x.ai/docs/models/grok-imagine-video", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59507,9 +59504,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59527,9 +59521,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59547,9 +59538,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 33e9ce94455..537db678f60 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -59487,9 +59487,6 @@ "output_cost_per_second": 0.05, "output_cost_per_second_480p": 0.05, "source": "https://docs.x.ai/docs/models/grok-imagine-video", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59507,9 +59504,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59527,9 +59521,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", @@ -59547,9 +59538,6 @@ "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", - "supported_endpoints": [ - "/v1/videos" - ], "supported_modalities": [ "text", "image", From 6c21be619441dbd24879e1f8a4b897c6dbd667fe Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 7 Sep 2026 12:15:06 -0700 Subject: [PATCH 059/411] fix(e2e): clean up batch files reliably and expire Azure inputs --- tests/e2e/batches/COVERAGE.md | 18 ++ tests/e2e/batches/batch_cleanup.py | 90 +++++++++ tests/e2e/batches/batch_client.py | 16 +- tests/e2e/batches/capabilities.py | 4 + tests/e2e/batches/conftest.py | 10 +- tests/e2e/batches/test_batch_cleanup.py | 187 ++++++++++++++++++ tests/e2e/batches/test_batches_e2e.py | 80 ++++---- .../test_managed_files_enforcement_e2e.py | 3 +- tests/e2e/lifecycle.py | 23 ++- 9 files changed, 381 insertions(+), 50 deletions(-) create mode 100644 tests/e2e/batches/batch_cleanup.py create mode 100644 tests/e2e/batches/test_batch_cleanup.py diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index 8a7b68511ec..899530dde2b 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -120,6 +120,24 @@ create traverse gateway -> gateway -> OpenAI (LIT-5347, PR #36240). The pin: nested managed ids round-trip retrieve. This self-chaining only needs the proxy to reach its own `PROXY_BASE_URL`, which holds both locally and on the e2e stage. +## Cleanup + +Batch teardown cancels active batches before deleting their input files and keys. +Raw file IDs from both `model_param` and `provider_fallback` uploads use the upload +provider when deleted. Model-encoded and managed file IDs route themselves + +File deletion and batch cancellation check their responses and retry transient +failures up to three times. Teardown attempts every registered cleanup before +reporting failures as test errors. Already deleted files and batches that are +terminal are safe to clean up again. Cancellation polls for up to ten minutes +before input deletion, because accepting cancellation does not finish it + +Azure input uploads request `expires_after` anchored to `created_at` with +`seconds=1209600`, and the lifecycle tests check the returned expiry. This is a +fallback for interrupted runs: immediate deletion remains the normal cleanup. +Azure's minimum supported native expiry is 14 days, so a three-day expiry cannot +be requested through its Files API + ## Terminal state + cost write-back (cross-run marker baton) The 24h completion window rules out submit-and-wait inside one run, so diff --git a/tests/e2e/batches/batch_cleanup.py b/tests/e2e/batches/batch_cleanup.py new file mode 100644 index 00000000000..a5b5e9bba37 --- /dev/null +++ b/tests/e2e/batches/batch_cleanup.py @@ -0,0 +1,90 @@ +from collections.abc import Callable +from time import monotonic, sleep +from typing import Final, Protocol + +from pydantic import BaseModel + +from batch_client import BatchObject, FileDeleteResponse +from e2e_http import NetworkError, RateLimitedError, Result, Success, UnknownApiError + +CLEANUP_DELAYS: Final = (1.0, 2.0, 4.0) +BATCH_TERMINAL_STATUSES: Final = frozenset({"completed", "failed", "expired", "cancelled"}) +BATCH_CANCEL_TIMEOUT_SECONDS: Final = 600.0 +BATCH_CANCEL_POLL_SECONDS: Final = 10.0 + + +class BatchCleanupClient(Protocol): + def delete_file(self, file_id: str, *, key: str, provider: str | None = None) -> Result[FileDeleteResponse]: ... + + def retrieve_batch(self, batch_id: str, *, key: str, provider: str | None = None) -> Result[BatchObject]: ... + + def cancel_batch(self, batch_id: str, *, key: str, provider: str | None = None) -> Result[BatchObject]: ... + + +def cleanup_result[R: BaseModel]( + action: Callable[[], Result[R]], *, wait: Callable[[float], None] = sleep +) -> Result[R]: + for delay, result in ((delay, action()) for delay in CLEANUP_DELAYS): + match result: + case NetworkError() | RateLimitedError(): + wait(delay) + case UnknownApiError(status_code=code) if code in {408, 429, 500, 502, 503, 504}: + wait(delay) + case _: + return result + return action() + + +def _require_cleanup_success[R: BaseModel](result: Result[R], operation: str) -> R: + match result: + case Success(data=data): + return data + case UnknownApiError(status_code=code): + raise AssertionError(f"{operation} failed: HTTP {code}") + case _: + raise AssertionError(f"{operation} failed: {result.kind}") + + +def cleanup_file(client: BatchCleanupClient, file_id: str, *, key: str, provider: str | None = None) -> None: + result: Final = cleanup_result(lambda: client.delete_file(file_id, key=key, provider=provider)) + if isinstance(result, UnknownApiError) and result.status_code == 404: + return + deleted: Final = _require_cleanup_success(result, f"Delete file {file_id}") + assert deleted.deleted, f"Delete file {file_id} did not confirm deletion" + + +def cleanup_batch( + client: BatchCleanupClient, + batch_id: str, + *, + key: str, + provider: str | None = None, + wait: Callable[[float], None] = sleep, + clock: Callable[[], float] = monotonic, +) -> None: + fetched: Final = _require_cleanup_success( + cleanup_result(lambda: client.retrieve_batch(batch_id, key=key, provider=provider)), + f"Retrieve batch {batch_id} for cleanup", + ) + if fetched.status in BATCH_TERMINAL_STATUSES: + return + if fetched.status != "cancelling": + result: Final = cleanup_result(lambda: client.cancel_batch(batch_id, key=key, provider=provider)) + if not (isinstance(result, UnknownApiError) and result.status_code in {400, 409}): + cancelled: Final = _require_cleanup_success(result, f"Cancel batch {batch_id}") + assert cancelled.status in BATCH_TERMINAL_STATUSES | {"cancelling"}, ( + f"Cancel batch {batch_id} left status {cancelled.status}" + ) + deadline: Final = clock() + BATCH_CANCEL_TIMEOUT_SECONDS + while True: + current = _require_cleanup_success( + cleanup_result(lambda: client.retrieve_batch(batch_id, key=key, provider=provider)), + f"Retrieve batch {batch_id} after cancellation", + ) + if current.status in BATCH_TERMINAL_STATUSES: + return + assert current.status == "cancelling", f"Cancel batch {batch_id} left status {current.status}" + assert clock() < deadline, ( + f"Batch {batch_id} cancellation did not finish within {BATCH_CANCEL_TIMEOUT_SECONDS}s" + ) + wait(BATCH_CANCEL_POLL_SECONDS) diff --git a/tests/e2e/batches/batch_client.py b/tests/e2e/batches/batch_client.py index 31e49f22450..84a902b0b11 100644 --- a/tests/e2e/batches/batch_client.py +++ b/tests/e2e/batches/batch_client.py @@ -13,8 +13,9 @@ co-located here because only this suite uses them. from __future__ import annotations from dataclasses import dataclass +from typing import Final, Literal -from pydantic import BaseModel +from pydantic import BaseModel, Field from proxy_client import ProxyClient from e2e_http import ( @@ -27,6 +28,18 @@ from e2e_http import ( from models import LiteLLMParamsBody UPLOAD_FILENAME = "batch_input.jsonl" +AZURE_FILE_EXPIRY_SECONDS: Final = 14 * 24 * 60 * 60 + + +class ExpiringFileUploadForm(FileUploadForm): + expires_after_anchor: Literal["created_at"] = Field(default="created_at", alias="expires_after[anchor]") + expires_after_seconds: int = Field(default=AZURE_FILE_EXPIRY_SECONDS, alias="expires_after[seconds]") + + +def batch_upload_form(provider: str, *, target_model_names: str | None = None) -> FileUploadForm: + if provider == "azure": + return ExpiringFileUploadForm(target_model_names=target_model_names) + return FileUploadForm(target_model_names=target_model_names) class FileObject(BaseModel): @@ -37,6 +50,7 @@ class FileObject(BaseModel): bytes: int | None = None status: str | None = None created_at: int | None = None + expires_at: int | None = None class FileList(BaseModel): diff --git a/tests/e2e/batches/capabilities.py b/tests/e2e/batches/capabilities.py index 1bcea0a61ee..17749c2fb87 100644 --- a/tests/e2e/batches/capabilities.py +++ b/tests/e2e/batches/capabilities.py @@ -108,6 +108,10 @@ class Capability: def id(self) -> str: return f"{self.provider}-{self.scenario}" + @property + def file_provider(self) -> str | None: + return self.provider if self.scenario in {"model_param", "provider_fallback"} else None + @property def jsonl_model(self) -> str: # Always the provider deployment name. Unified routes via diff --git a/tests/e2e/batches/conftest.py b/tests/e2e/batches/conftest.py index 3b133fab680..91a365b6b92 100644 --- a/tests/e2e/batches/conftest.py +++ b/tests/e2e/batches/conftest.py @@ -13,7 +13,7 @@ the proxy config. from __future__ import annotations import os -from typing import Iterator +from typing import Final, Iterator import pytest @@ -21,6 +21,7 @@ from batch_client import BatchClient, build_client from capabilities import PROVIDERS from e2e_config import MANAGED_FILES_OPT_IN_ENV from e2e_http import NoBody +from lifecycle import ResourceManager from proxy_client import ProxyClient @@ -52,6 +53,13 @@ def client(proxy: ProxyClient) -> BatchClient: return build_client(proxy) +@pytest.fixture +def resources(client: BatchClient) -> Iterator[ResourceManager]: + manager: Final = ResourceManager(client=client.proxy, strict_cleanup=True) + yield manager + manager.teardown() + + @pytest.fixture(scope="session") def batch_deployments(client: BatchClient) -> Iterator[None]: probe = client.proxy.probe("/health/liveliness", params=NoBody()) diff --git a/tests/e2e/batches/test_batch_cleanup.py b/tests/e2e/batches/test_batch_cleanup.py new file mode 100644 index 00000000000..8ebfd750360 --- /dev/null +++ b/tests/e2e/batches/test_batch_cleanup.py @@ -0,0 +1,187 @@ +from builtins import ExceptionGroup +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import Final + +import pytest + +from batch_cleanup import BATCH_CANCEL_TIMEOUT_SECONDS, CLEANUP_DELAYS, cleanup_batch, cleanup_file, cleanup_result +from batch_client import AZURE_FILE_EXPIRY_SECONDS, BatchObject, FileDeleteResponse, batch_upload_form +from capabilities import CAPABILITIES, Capability +from e2e_http import NetworkError, RateLimitedError, Result, Success, UnknownApiError +from lifecycle import ResourceManager +from models import KeyGenerateBody + + +@dataclass +class CleanupClient: + files: Iterator[Result[FileDeleteResponse]] = field(default_factory=lambda: iter(())) + batches: Iterator[Result[BatchObject]] = field(default_factory=lambda: iter(())) + cancellations: Iterator[Result[BatchObject]] = field(default_factory=lambda: iter(())) + calls: list[str] = field(default_factory=list) + + def delete_file(self, file_id: str, *, key: str, provider: str | None = None) -> Result[FileDeleteResponse]: + self.calls.append(f"delete {provider} {file_id}") + return next(self.files) + + def retrieve_batch(self, batch_id: str, *, key: str, provider: str | None = None) -> Result[BatchObject]: + self.calls.append(f"retrieve {provider} {batch_id}") + return next(self.batches) + + def cancel_batch(self, batch_id: str, *, key: str, provider: str | None = None) -> Result[BatchObject]: + self.calls.append(f"cancel {provider} {batch_id}") + return next(self.cancellations) + + def generate_key(self, body: KeyGenerateBody) -> str: + return "test-key" + + def delete_key(self, key: str) -> None: + self.calls.append(f"delete key {key}") + + def delete_customers(self, user_ids: list[str]) -> None: + self.calls.append(f"delete customers {user_ids}") + + +def batch(status: str) -> Success[BatchObject]: + return Success(status_code=200, data=BatchObject(id="batch-1", status=status)) + + +def deleted_file(*, deleted: bool = True) -> Success[FileDeleteResponse]: + return Success(status_code=200, data=FileDeleteResponse(id="file-1", deleted=deleted)) + + +class TestFileCleanup: + @pytest.mark.parametrize("cap", CAPABILITIES, ids=[cap.id for cap in CAPABILITIES]) + def test_deletes_raw_files_through_the_upload_provider(self, cap: Capability) -> None: + client: Final = CleanupClient(files=iter((deleted_file(),))) + cleanup_file(client, "file-1", key="test-key", provider=cap.file_provider) + expected_provider: Final = cap.provider if cap.scenario in {"model_param", "provider_fallback"} else None + assert client.calls == [f"delete {expected_provider} file-1"] + + def test_failed_delete_is_reported_after_remaining_resources_are_cleaned(self) -> None: + client: Final = CleanupClient(files=iter((UnknownApiError(status_code=403, body="secret response"),))) + manager: Final = ResourceManager(client=client, strict_cleanup=True) + key: Final = manager.key() + manager.defer(lambda: cleanup_file(client, "file-1", key=key, provider="azure")) + with pytest.raises(ExceptionGroup) as caught: + manager.teardown() + assert client.calls == ["delete azure file-1", "delete key test-key"] + assert len(caught.value.exceptions) == 1 + assert str(caught.value.exceptions[0]) == "Delete file file-1 failed: HTTP 403" + + def test_success_response_must_confirm_deletion(self) -> None: + client: Final = CleanupClient(files=iter((deleted_file(deleted=False),))) + with pytest.raises(AssertionError, match="did not confirm deletion"): + cleanup_file(client, "file-1", key="test-key") + + def test_cleanup_is_idempotent_when_file_is_already_deleted(self) -> None: + client: Final = CleanupClient(files=iter((UnknownApiError(status_code=404, body="missing"),))) + cleanup_file(client, "file-1", key="test-key", provider="azure") + assert client.calls == ["delete azure file-1"] + + def test_default_resource_cleanup_keeps_existing_best_effort_behavior(self) -> None: + client: Final = CleanupClient(files=iter((UnknownApiError(status_code=403, body="forbidden"),))) + manager: Final = ResourceManager(client=client) + key: Final = manager.key() + manager.defer(lambda: cleanup_file(client, "file-1", key=key)) + manager.teardown() + assert client.calls == ["delete None file-1", "delete key test-key"] + + +class TestCleanupRetries: + @pytest.mark.parametrize( + "failure", + [NetworkError(message="offline"), RateLimitedError(), UnknownApiError(status_code=503, body="unavailable")], + ) + def test_transient_error_retries_and_returns_success(self, failure: Result[FileDeleteResponse]) -> None: + outcomes: Final = iter((failure, deleted_file())) + delays: Final[list[float]] = [] + result: Final[Result[FileDeleteResponse]] = cleanup_result(lambda: next(outcomes), wait=delays.append) + assert isinstance(result, Success) and result.data.deleted + assert delays == [1.0] + + def test_persistent_error_has_bounded_retries(self) -> None: + failure: Final = UnknownApiError(status_code=503, body="unavailable") + outcomes: Final[Iterator[Result[FileDeleteResponse]]] = iter((failure,) * (len(CLEANUP_DELAYS) + 1)) + delays: Final[list[float]] = [] + result: Final[Result[FileDeleteResponse]] = cleanup_result(lambda: next(outcomes), wait=delays.append) + assert result is failure + assert tuple(delays) == CLEANUP_DELAYS + assert next(outcomes, None) is None + + def test_permanent_error_is_not_retried(self) -> None: + failure: Final = UnknownApiError(status_code=403, body="forbidden") + outcomes: Final = iter((failure, deleted_file())) + delays: Final[list[float]] = [] + assert cleanup_result(lambda: next(outcomes), wait=delays.append) is failure + assert delays == [] + assert isinstance(next(outcomes), Success) + + +class TestBatchCancellation: + def test_cancelling_batch_is_polled_until_terminal_without_cancelling_again(self) -> None: + client: Final = CleanupClient(batches=iter((batch("cancelling"), batch("cancelling"), batch("cancelled")))) + delays: Final[list[float]] = [] + cleanup_batch(client, "batch-1", key="test-key", wait=delays.append) + assert client.calls == ["retrieve None batch-1"] * 3 + assert delays == [10.0] + + def test_cancellation_timeout_is_reported_but_file_and_key_cleanup_still_run(self) -> None: + client: Final = CleanupClient( + batches=iter((batch("cancelling"), batch("cancelling"))), files=iter((deleted_file(),)) + ) + ticks: Final = iter((0.0, BATCH_CANCEL_TIMEOUT_SECONDS)) + manager: Final = ResourceManager(client=client, strict_cleanup=True) + key: Final = manager.key() + manager.defer(lambda: cleanup_file(client, "file-1", key=key)) + manager.defer(lambda: cleanup_batch(client, "batch-1", key=key, clock=lambda: next(ticks))) + with pytest.raises(ExceptionGroup) as caught: + manager.teardown() + assert "cancellation did not finish" in str(caught.value.exceptions[0]) + assert client.calls == [ + "retrieve None batch-1", + "retrieve None batch-1", + "delete None file-1", + "delete key test-key", + ] + + @pytest.mark.parametrize("status", ["completed", "failed", "expired", "cancelled"]) + def test_inactive_batch_needs_no_cancellation(self, status: str) -> None: + client: Final = CleanupClient(batches=iter((batch(status),))) + cleanup_batch(client, "batch-1", key="test-key") + assert client.calls == ["retrieve None batch-1"] + + def test_active_batch_is_cancelled_through_its_provider(self) -> None: + client: Final = CleanupClient( + batches=iter((batch("in_progress"), batch("cancelled"))), cancellations=iter((batch("cancelling"),)) + ) + cleanup_batch(client, "batch-1", key="test-key", provider="azure") + assert client.calls == ["retrieve azure batch-1", "cancel azure batch-1", "retrieve azure batch-1"] + + @pytest.mark.parametrize("status", ["completed", "in_progress"]) + def test_cancellation_conflict_is_accepted_only_when_batch_became_inactive(self, status: str) -> None: + client: Final = CleanupClient( + batches=iter((batch("in_progress"), batch(status))), + cancellations=iter((UnknownApiError(status_code=409, body="conflict"),)), + ) + if status == "completed": + cleanup_batch(client, "batch-1", key="test-key") + else: + with pytest.raises(AssertionError, match="Cancel batch batch-1 left status in_progress"): + cleanup_batch(client, "batch-1", key="test-key") + assert client.calls == ["retrieve None batch-1", "cancel None batch-1", "retrieve None batch-1"] + + +class TestAzureFileExpiry: + def test_azure_form_serializes_native_expiry_for_the_proxy(self) -> None: + form: Final = batch_upload_form("azure", target_model_names="azure-test") + assert form.model_dump(by_alias=True, exclude_none=True) == { + "purpose": "batch", + "target_model_names": "azure-test", + "expires_after[anchor]": "created_at", + "expires_after[seconds]": AZURE_FILE_EXPIRY_SECONDS, + } + + @pytest.mark.parametrize("provider", ["openai", "vertex_ai", "bedrock"]) + def test_other_providers_keep_their_existing_upload_fields(self, provider: str) -> None: + assert batch_upload_form(provider).model_dump(by_alias=True, exclude_none=True) == {"purpose": "batch"} diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index ed7cf656d01..1b4a6ed266f 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -21,14 +21,16 @@ import os import re import time from datetime import datetime, timedelta, timezone -from typing import Callable import pytest from pydantic import BaseModel from e2e_config import PROXY_BASE_URL, unique_marker +from batch_cleanup import cleanup_batch, cleanup_file from batch_client import ( + AZURE_FILE_EXPIRY_SECONDS, + batch_upload_form, UPLOAD_FILENAME, BatchClient, BatchCreateBody, @@ -155,19 +157,19 @@ def upload_for_scenario( if cap.scenario == "encoded": return client.upload_file( content=content, - form=FileUploadForm(purpose="batch"), + form=batch_upload_form(cap.provider), model=cap.model, key=key, ) if cap.scenario == "unified": return client.upload_file( content=content, - form=FileUploadForm(purpose="batch", target_model_names=cap.model), + form=batch_upload_form(cap.provider, target_model_names=cap.model), key=key, ) return client.upload_file( content=content, - form=FileUploadForm(purpose="batch"), + form=batch_upload_form(cap.provider), key=key, provider=cap.provider, ) @@ -188,20 +190,11 @@ def create_for_scenario( def op_provider(cap: Capability) -> str | None: - """provider_fallback ids are raw, so retrieve/cancel/list/delete need the provider + """provider_fallback batch ids are raw, so retrieve/cancel/list need the provider hint; the other scenarios encode it into the id and route automatically.""" return cap.provider if cap.scenario == "provider_fallback" else None -def quietly(action: Callable[[], object]) -> Callable[[], None]: - """Adapt a value-returning call into a best-effort cleanup the teardown can run.""" - - def run() -> None: - action() - - return run - - def assert_file_object(file: FileObject, *, provider: str) -> None: assert file.object == "file", f"file.object={file.object!r}" assert file.purpose == "batch", f"file.purpose={file.purpose!r}" @@ -209,6 +202,10 @@ def assert_file_object(file: FileObject, *, provider: str) -> None: if provider != "bedrock": assert file.bytes > 0, f"file.bytes={file.bytes!r}" assert file.status, "file.status missing" + if provider == "azure": + assert file.expires_at is not None, "Azure batch input has no automatic expiry" + assert file.created_at is not None + assert file.expires_at - file.created_at == AZURE_FILE_EXPIRY_SECONDS assert ( file.created_at is not None and file.created_at > 0 ), "file.created_at missing" @@ -249,7 +246,7 @@ def test_batch_lifecycle( file = unwrap(upload_for_scenario(client, cap, render_jsonl(cap.jsonl_model), key)) resources.defer( - quietly(lambda: client.delete_file(file.id, key=key, provider=provider)) + lambda: cleanup_file(client, file.id, key=key, provider=cap.file_provider) ) assert_file_object(file, provider=cap.provider) assert matches_id_shape( @@ -260,7 +257,7 @@ def test_batch_lifecycle( require_successful_call(created) batch = BatchObject.model_validate_json(created.body) resources.defer( - quietly(lambda: client.cancel_batch(batch.id, key=key, provider=provider)) + lambda: cleanup_batch(client, batch.id, key=key, provider=provider) ) assert batch.id, f"create returned no batch id (body={created.body[:200]})" @@ -339,7 +336,7 @@ def test_batch_key_model_access_denied( denied_upload = client.upload_file( content=render_jsonl(AZURE_BATCH_MODEL), - form=FileUploadForm(purpose="batch"), + form=batch_upload_form("azure"), model=AZURE_BATCH_MODEL, key=key, ) @@ -356,7 +353,7 @@ def test_batch_key_model_access_denied( ) ).id resources.defer( - quietly(lambda: client.delete_file(raw_file, key=key, provider="openai")) + lambda: cleanup_file(client, raw_file, key=key, provider="openai") ) denied_create = client.create_batch( @@ -383,6 +380,7 @@ def test_file_upload_and_delete_outputs( key=key, ) ) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) assert_file_object(file, provider="openai") deleted = unwrap(client.delete_file(file.id, key=key)) @@ -458,12 +456,12 @@ def test_rate_limited_batch_create_leaves_no_unattributed_spend_row( key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) require_successful_call(created) batch = BatchObject.model_validate_json(created.body) - resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + resources.defer(lambda: cleanup_batch(client, batch.id, key=key)) _ = client.proxy.poll_logs_for_key(key, min_rows=1) @@ -517,7 +515,7 @@ class TestBatchFileContent: key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) assert file.id downloaded = client.proxy.transport.download( @@ -559,11 +557,11 @@ class TestBatchFileContent: file = unwrap( client.upload_file( content=payload, - form=FileUploadForm(purpose="batch", target_model_names=provider.model), + form=batch_upload_form(provider.name, target_model_names=provider.model), key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) assert_file_object(file, provider=provider.name) assert is_managed_id(file.id), ( f"{provider.name}: unified upload must return a managed file id, got {file.id!r}" @@ -626,7 +624,7 @@ class TestOpenAIFiles: ) ) resources.defer( - quietly(lambda: client.delete_file(file.id, key=key, provider="openai")) + lambda: cleanup_file(client, file.id, key=key, provider="openai") ) listed = unwrap(client.list_files(key=key)) @@ -690,7 +688,7 @@ class TestOpenAIFiles: key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) fetched = unwrap(client.retrieve_file(file.id, key=key)) assert fetched.id == file.id, "retrieve must echo the uploaded file id" @@ -760,7 +758,7 @@ class TestBatchRateLimitErrorMapping: key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) @@ -813,7 +811,7 @@ class TestBatchEnqueuedTokenLimit: key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) return file def _generate_enqueued_key( @@ -861,7 +859,7 @@ class TestBatchEnqueuedTokenLimit: ) require_successful_call(created) batch = BatchObject.model_validate_json(created.body) - resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + resources.defer(lambda: cleanup_batch(client, batch.id, key=key)) @pytest.mark.covers( "quota_management.ratelimit.batch_enqueued_tokens.blocks_when_exhausted", @@ -904,7 +902,7 @@ class TestBatchEnqueuedTokenLimit: first = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) require_successful_call(first) first_batch = BatchObject.model_validate_json(first.body) - resources.defer(quietly(lambda: client.cancel_batch(first_batch.id, key=key))) + resources.defer(lambda: cleanup_batch(client, first_batch.id, key=key)) blocked = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) assert blocked.status_code == 429, ( @@ -928,7 +926,7 @@ class TestBatchEnqueuedTokenLimit: ) require_successful_call(retried) retry_batch = BatchObject.model_validate_json(retried.body) - resources.defer(quietly(lambda: client.cancel_batch(retry_batch.id, key=key))) + resources.defer(lambda: cleanup_batch(client, retry_batch.id, key=key)) ASSUME_ROLE_RAW_MODEL = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" @@ -984,13 +982,13 @@ class TestBedrockBatchAssumeRole: key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) assert_file_object(file, provider="bedrock") created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) require_successful_call(created) batch = BatchObject.model_validate_json(created.body) - resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + resources.defer(lambda: cleanup_batch(client, batch.id, key=key)) assert batch.id, f"assume-role create returned no batch id: {created.body[:200]}" assert is_managed_id(batch.id), ( @@ -1044,7 +1042,7 @@ class TestGeminiFiles: key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) assert_file_object(file, provider="gemini") assert file.id, "gemini file upload returned no id" @@ -1099,13 +1097,13 @@ class TestHostedVllmBatch: key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) assert_file_object(file, provider="hosted_vllm") created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) require_successful_call(created) batch = BatchObject.model_validate_json(created.body) - resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + resources.defer(lambda: cleanup_batch(client, batch.id, key=key)) assert batch.id, f"hosted_vllm create returned no batch id: {created.body[:200]}" assert batch.status in CREATED_BATCH_STATUSES, ( @@ -1192,7 +1190,7 @@ class TestBatchFailurePaths: key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) require_successful_call(created) @@ -1243,12 +1241,12 @@ class TestBatchFailurePaths: file = unwrap( client.upload_file( content=render_jsonl(AZURE_BATCH_RAW_MODEL), - form=FileUploadForm(purpose="batch"), + form=batch_upload_form("azure"), model=AZURE_BATCH_MODEL, key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) assert decoded_model_from_id(file.id) == AZURE_BATCH_MODEL, ( f"upload did not encode the azure deployment into the file id: {file.id!r}" ) @@ -1258,7 +1256,7 @@ class TestBatchFailurePaths: ) require_successful_call(created) batch = BatchObject.model_validate_json(created.body) - resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + resources.defer(lambda: cleanup_batch(client, batch.id, key=key)) assert decoded_model_from_id(batch.id) == AZURE_BATCH_MODEL, ( "create with a foreign encoded file id must route by the file's embedded model, " @@ -1307,7 +1305,7 @@ class TestBatchSecondHop: key=key, ) ) - resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) assert is_managed_id(file.id), ( f"second-hop unified upload must return a managed file id, got {file.id!r}" ) @@ -1315,7 +1313,7 @@ class TestBatchSecondHop: created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) require_successful_call(created) batch = BatchObject.model_validate_json(created.body) - resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + resources.defer(lambda: cleanup_batch(client, batch.id, key=key)) assert is_managed_id(batch.id), ( f"second-hop create must return a managed batch id, got {batch.id!r}" diff --git a/tests/e2e/batches/test_managed_files_enforcement_e2e.py b/tests/e2e/batches/test_managed_files_enforcement_e2e.py index 7ad0b16adc3..4f703cf0fdc 100644 --- a/tests/e2e/batches/test_managed_files_enforcement_e2e.py +++ b/tests/e2e/batches/test_managed_files_enforcement_e2e.py @@ -21,6 +21,7 @@ from typing import Iterator import pytest from batch_client import BatchClient, FileObject +from batch_cleanup import cleanup_file from capabilities import batch_model_name, is_managed_id, openai_batch_params from e2e_config import unique_marker from e2e_http import FileUploadForm, Result, UnknownApiError, unwrap @@ -108,7 +109,7 @@ def test_cross_user_managed_id_denied_owner_allowed( key=owner_key, ) ) - resources.defer(lambda: client.delete_file(uploaded.id, key=owner_key)) + resources.defer(lambda: cleanup_file(client, uploaded.id, key=owner_key)) assert is_managed_id(uploaded.id), f"expected a managed unified file id, got {uploaded.id}" denied = client.retrieve_file(uploaded.id, key=other_key) diff --git a/tests/e2e/lifecycle.py b/tests/e2e/lifecycle.py index c9a67ebdb8c..eb9704d4dcb 100644 --- a/tests/e2e/lifecycle.py +++ b/tests/e2e/lifecycle.py @@ -8,8 +8,9 @@ ResourceManager; the test registers a cleanup for every resource it creates, and the fixture's teardown releases them all even when the test body raises. """ +from builtins import ExceptionGroup from dataclasses import dataclass, field -from typing import Callable, List, Protocol, runtime_checkable +from typing import Callable, Final, List, Protocol, runtime_checkable from proxy_client import ProxyClient from models import KeyGenerateBody @@ -52,6 +53,7 @@ class ResourceManager: """ client: ResourceClient + strict_cleanup: bool = False _cleanups: List[Callable[[], object]] = field( default_factory=list ) # mutable-ok: append-only teardown registry @@ -82,8 +84,17 @@ class ResourceManager: return customer_id def teardown(self) -> None: - for cleanup in reversed(self._cleanups): - try: - cleanup() - except Exception: - pass # best-effort: a failed cleanup must not block the rest + failures: Final = tuple( + failure for cleanup in reversed(self._cleanups) + if (failure := _run_cleanup(cleanup)) is not None + ) + if failures and self.strict_cleanup: + raise ExceptionGroup("Resource cleanup failed", failures) + + +def _run_cleanup(cleanup: Callable[[], object]) -> Exception | None: + try: + cleanup() + except Exception as exc: + return exc + return None From a56c60e8924f6f956200c816142fb0cb994fa3ed Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 7 Sep 2026 12:27:51 -0700 Subject: [PATCH 060/411] fix(e2e): wait for managed batch cancellation before deleting inputs --- tests/e2e/batches/COVERAGE.md | 5 +++-- tests/e2e/batches/batch_cleanup.py | 10 +++++++++- tests/e2e/batches/test_batch_cleanup.py | 14 ++++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index 899530dde2b..69ba9d781ec 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -129,8 +129,9 @@ provider when deleted. Model-encoded and managed file IDs route themselves File deletion and batch cancellation check their responses and retry transient failures up to three times. Teardown attempts every registered cleanup before reporting failures as test errors. Already deleted files and batches that are -terminal are safe to clean up again. Cancellation polls for up to ten minutes -before input deletion, because accepting cancellation does not finish it +terminal are safe to clean up again. Managed batch cancellation polls for up to eleven minutes +before input deletion: the ten-minute provider window plus a propagation margin. +Raw and model-encoded inputs can be deleted after cancellation is accepted Azure input uploads request `expires_after` anchored to `created_at` with `seconds=1209600`, and the lifecycle tests check the returned expiry. This is a diff --git a/tests/e2e/batches/batch_cleanup.py b/tests/e2e/batches/batch_cleanup.py index a5b5e9bba37..dd79c776758 100644 --- a/tests/e2e/batches/batch_cleanup.py +++ b/tests/e2e/batches/batch_cleanup.py @@ -5,11 +5,12 @@ from typing import Final, Protocol from pydantic import BaseModel from batch_client import BatchObject, FileDeleteResponse +from capabilities import is_managed_id from e2e_http import NetworkError, RateLimitedError, Result, Success, UnknownApiError CLEANUP_DELAYS: Final = (1.0, 2.0, 4.0) BATCH_TERMINAL_STATUSES: Final = frozenset({"completed", "failed", "expired", "cancelled"}) -BATCH_CANCEL_TIMEOUT_SECONDS: Final = 600.0 +BATCH_CANCEL_TIMEOUT_SECONDS: Final = 660.0 BATCH_CANCEL_POLL_SECONDS: Final = 10.0 @@ -62,12 +63,15 @@ def cleanup_batch( wait: Callable[[float], None] = sleep, clock: Callable[[], float] = monotonic, ) -> None: + needs_terminal_state: Final = is_managed_id(batch_id) fetched: Final = _require_cleanup_success( cleanup_result(lambda: client.retrieve_batch(batch_id, key=key, provider=provider)), f"Retrieve batch {batch_id} for cleanup", ) if fetched.status in BATCH_TERMINAL_STATUSES: return + if fetched.status == "cancelling" and not needs_terminal_state: + return if fetched.status != "cancelling": result: Final = cleanup_result(lambda: client.cancel_batch(batch_id, key=key, provider=provider)) if not (isinstance(result, UnknownApiError) and result.status_code in {400, 409}): @@ -75,6 +79,8 @@ def cleanup_batch( assert cancelled.status in BATCH_TERMINAL_STATUSES | {"cancelling"}, ( f"Cancel batch {batch_id} left status {cancelled.status}" ) + if not needs_terminal_state: + return deadline: Final = clock() + BATCH_CANCEL_TIMEOUT_SECONDS while True: current = _require_cleanup_success( @@ -84,6 +90,8 @@ def cleanup_batch( if current.status in BATCH_TERMINAL_STATUSES: return assert current.status == "cancelling", f"Cancel batch {batch_id} left status {current.status}" + if not needs_terminal_state: + return assert clock() < deadline, ( f"Batch {batch_id} cancellation did not finish within {BATCH_CANCEL_TIMEOUT_SECONDS}s" ) diff --git a/tests/e2e/batches/test_batch_cleanup.py b/tests/e2e/batches/test_batch_cleanup.py index 8ebfd750360..9715370d9b7 100644 --- a/tests/e2e/batches/test_batch_cleanup.py +++ b/tests/e2e/batches/test_batch_cleanup.py @@ -12,6 +12,8 @@ from e2e_http import NetworkError, RateLimitedError, Result, Success, UnknownApi from lifecycle import ResourceManager from models import KeyGenerateBody +MANAGED_BATCH_ID: Final = "bGl0ZWxsbV9wcm94eTtiYXRjaC0x" + @dataclass class CleanupClient: @@ -122,8 +124,8 @@ class TestBatchCancellation: def test_cancelling_batch_is_polled_until_terminal_without_cancelling_again(self) -> None: client: Final = CleanupClient(batches=iter((batch("cancelling"), batch("cancelling"), batch("cancelled")))) delays: Final[list[float]] = [] - cleanup_batch(client, "batch-1", key="test-key", wait=delays.append) - assert client.calls == ["retrieve None batch-1"] * 3 + cleanup_batch(client, MANAGED_BATCH_ID, key="test-key", wait=delays.append) + assert client.calls == [f"retrieve None {MANAGED_BATCH_ID}"] * 3 assert delays == [10.0] def test_cancellation_timeout_is_reported_but_file_and_key_cleanup_still_run(self) -> None: @@ -134,13 +136,13 @@ class TestBatchCancellation: manager: Final = ResourceManager(client=client, strict_cleanup=True) key: Final = manager.key() manager.defer(lambda: cleanup_file(client, "file-1", key=key)) - manager.defer(lambda: cleanup_batch(client, "batch-1", key=key, clock=lambda: next(ticks))) + manager.defer(lambda: cleanup_batch(client, MANAGED_BATCH_ID, key=key, clock=lambda: next(ticks))) with pytest.raises(ExceptionGroup) as caught: manager.teardown() assert "cancellation did not finish" in str(caught.value.exceptions[0]) assert client.calls == [ - "retrieve None batch-1", - "retrieve None batch-1", + f"retrieve None {MANAGED_BATCH_ID}", + f"retrieve None {MANAGED_BATCH_ID}", "delete None file-1", "delete key test-key", ] @@ -156,7 +158,7 @@ class TestBatchCancellation: batches=iter((batch("in_progress"), batch("cancelled"))), cancellations=iter((batch("cancelling"),)) ) cleanup_batch(client, "batch-1", key="test-key", provider="azure") - assert client.calls == ["retrieve azure batch-1", "cancel azure batch-1", "retrieve azure batch-1"] + assert client.calls == ["retrieve azure batch-1", "cancel azure batch-1"] @pytest.mark.parametrize("status", ["completed", "in_progress"]) def test_cancellation_conflict_is_accepted_only_when_batch_became_inactive(self, status: str) -> None: From 6bdf206cc5228e85b47c908a2f4977374656289b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 7 Sep 2026 12:48:07 -0700 Subject: [PATCH 061/411] fix(e2e): accept managed file deletion responses --- tests/e2e/batches/COVERAGE.md | 3 +++ tests/e2e/batches/batch_cleanup.py | 4 +++- tests/e2e/batches/batch_client.py | 2 +- tests/e2e/batches/test_batch_cleanup.py | 15 +++++++++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index 69ba9d781ec..f95ea1f2649 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -139,6 +139,9 @@ fallback for interrupted runs: immediate deletion remains the normal cleanup. Azure's minimum supported native expiry is 14 days, so a three-day expiry cannot be requested through its Files API +The Azure entry in `files_settings` must use `api_version: 2025-04-01-preview` +for raw uploads to honor expiry, matching the batch deployment's API version + ## Terminal state + cost write-back (cross-run marker baton) The 24h completion window rules out submit-and-wait inside one run, so diff --git a/tests/e2e/batches/batch_cleanup.py b/tests/e2e/batches/batch_cleanup.py index dd79c776758..3fd6802f696 100644 --- a/tests/e2e/batches/batch_cleanup.py +++ b/tests/e2e/batches/batch_cleanup.py @@ -51,7 +51,9 @@ def cleanup_file(client: BatchCleanupClient, file_id: str, *, key: str, provider if isinstance(result, UnknownApiError) and result.status_code == 404: return deleted: Final = _require_cleanup_success(result, f"Delete file {file_id}") - assert deleted.deleted, f"Delete file {file_id} did not confirm deletion" + assert deleted.deleted is True or ( + deleted.deleted is None and is_managed_id(file_id) and deleted.id == file_id and deleted.object == "file" + ), f"Delete file {file_id} did not confirm deletion" def cleanup_batch( diff --git a/tests/e2e/batches/batch_client.py b/tests/e2e/batches/batch_client.py index 84a902b0b11..c9c77e1f12e 100644 --- a/tests/e2e/batches/batch_client.py +++ b/tests/e2e/batches/batch_client.py @@ -99,7 +99,7 @@ class BatchList(BaseModel): class FileDeleteResponse(BaseModel): id: str object: str | None = None - deleted: bool + deleted: bool | None = None class BatchCreateBody(BaseModel): diff --git a/tests/e2e/batches/test_batch_cleanup.py b/tests/e2e/batches/test_batch_cleanup.py index 9715370d9b7..15dead6d36d 100644 --- a/tests/e2e/batches/test_batch_cleanup.py +++ b/tests/e2e/batches/test_batch_cleanup.py @@ -12,6 +12,7 @@ from e2e_http import NetworkError, RateLimitedError, Result, Success, UnknownApi from lifecycle import ResourceManager from models import KeyGenerateBody +MANAGED_FILE_ID: Final = "bGl0ZWxsbV9wcm94eTtmaWxlLTE=" MANAGED_BATCH_ID: Final = "bGl0ZWxsbV9wcm94eTtiYXRjaC0x" @@ -53,6 +54,20 @@ def deleted_file(*, deleted: bool = True) -> Success[FileDeleteResponse]: class TestFileCleanup: + def test_managed_delete_accepts_the_deleted_file_object(self) -> None: + response: Final = Success( + status_code=200, data=FileDeleteResponse.model_validate({"id": MANAGED_FILE_ID, "object": "file"}) + ) + client: Final = CleanupClient(files=iter((response,))) + cleanup_file(client, MANAGED_FILE_ID, key="test-key") + assert client.calls == [f"delete None {MANAGED_FILE_ID}"] + + @pytest.mark.parametrize("file_id", ["file-1", MANAGED_FILE_ID]) + def test_a_success_status_without_a_deletion_confirmation_is_rejected(self, file_id: str) -> None: + client: Final = CleanupClient(files=iter((Success(status_code=200, data=FileDeleteResponse(id=file_id)),))) + with pytest.raises(AssertionError, match="did not confirm deletion"): + cleanup_file(client, file_id, key="test-key") + @pytest.mark.parametrize("cap", CAPABILITIES, ids=[cap.id for cap in CAPABILITIES]) def test_deletes_raw_files_through_the_upload_provider(self, cap: Capability) -> None: client: Final = CleanupClient(files=iter((deleted_file(),))) From a096dd615c71e40be9473338ffeb8d1c17284f51 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 7 Sep 2026 14:39:25 -0700 Subject: [PATCH 062/411] refactor(e2e): use immutable batch cleanup test expectations --- tests/e2e/batches/batch_cleanup.py | 7 +- tests/e2e/batches/test_batch_cleanup.py | 137 ++++++++++++++++-------- 2 files changed, 96 insertions(+), 48 deletions(-) diff --git a/tests/e2e/batches/batch_cleanup.py b/tests/e2e/batches/batch_cleanup.py index 3fd6802f696..722df0c29bc 100644 --- a/tests/e2e/batches/batch_cleanup.py +++ b/tests/e2e/batches/batch_cleanup.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from itertools import count from time import monotonic, sleep from typing import Final, Protocol @@ -84,11 +85,13 @@ def cleanup_batch( if not needs_terminal_state: return deadline: Final = clock() + BATCH_CANCEL_TIMEOUT_SECONDS - while True: - current = _require_cleanup_success( + for current in ( + _require_cleanup_success( cleanup_result(lambda: client.retrieve_batch(batch_id, key=key, provider=provider)), f"Retrieve batch {batch_id} after cancellation", ) + for _ in count() + ): if current.status in BATCH_TERMINAL_STATUSES: return assert current.status == "cancelling", f"Cancel batch {batch_id} left status {current.status}" diff --git a/tests/e2e/batches/test_batch_cleanup.py b/tests/e2e/batches/test_batch_cleanup.py index 15dead6d36d..66b7079ccc2 100644 --- a/tests/e2e/batches/test_batch_cleanup.py +++ b/tests/e2e/batches/test_batch_cleanup.py @@ -16,33 +16,44 @@ MANAGED_FILE_ID: Final = "bGl0ZWxsbV9wcm94eTtmaWxlLTE=" MANAGED_BATCH_ID: Final = "bGl0ZWxsbV9wcm94eTtiYXRjaC0x" -@dataclass +@dataclass(frozen=True, slots=True) +class ExpectedCalls[T]: + values: Iterator[T] + + def __call__(self, value: T) -> None: + assert next(self.values, None) == value + + def assert_done(self) -> None: + assert tuple(self.values) == () + + +@dataclass(frozen=True, slots=True) class CleanupClient: + calls: ExpectedCalls[str] files: Iterator[Result[FileDeleteResponse]] = field(default_factory=lambda: iter(())) batches: Iterator[Result[BatchObject]] = field(default_factory=lambda: iter(())) cancellations: Iterator[Result[BatchObject]] = field(default_factory=lambda: iter(())) - calls: list[str] = field(default_factory=list) def delete_file(self, file_id: str, *, key: str, provider: str | None = None) -> Result[FileDeleteResponse]: - self.calls.append(f"delete {provider} {file_id}") + self.calls(f"delete {provider} {file_id}") return next(self.files) def retrieve_batch(self, batch_id: str, *, key: str, provider: str | None = None) -> Result[BatchObject]: - self.calls.append(f"retrieve {provider} {batch_id}") + self.calls(f"retrieve {provider} {batch_id}") return next(self.batches) def cancel_batch(self, batch_id: str, *, key: str, provider: str | None = None) -> Result[BatchObject]: - self.calls.append(f"cancel {provider} {batch_id}") + self.calls(f"cancel {provider} {batch_id}") return next(self.cancellations) def generate_key(self, body: KeyGenerateBody) -> str: return "test-key" def delete_key(self, key: str) -> None: - self.calls.append(f"delete key {key}") + self.calls(f"delete key {key}") def delete_customers(self, user_ids: list[str]) -> None: - self.calls.append(f"delete customers {user_ids}") + self.calls(f"delete customers {user_ids}") def batch(status: str) -> Success[BatchObject]: @@ -58,51 +69,71 @@ class TestFileCleanup: response: Final = Success( status_code=200, data=FileDeleteResponse.model_validate({"id": MANAGED_FILE_ID, "object": "file"}) ) - client: Final = CleanupClient(files=iter((response,))) + client: Final = CleanupClient( + calls=ExpectedCalls(iter((f"delete None {MANAGED_FILE_ID}",))), files=iter((response,)) + ) cleanup_file(client, MANAGED_FILE_ID, key="test-key") - assert client.calls == [f"delete None {MANAGED_FILE_ID}"] + client.calls.assert_done() @pytest.mark.parametrize("file_id", ["file-1", MANAGED_FILE_ID]) def test_a_success_status_without_a_deletion_confirmation_is_rejected(self, file_id: str) -> None: - client: Final = CleanupClient(files=iter((Success(status_code=200, data=FileDeleteResponse(id=file_id)),))) + client: Final = CleanupClient( + calls=ExpectedCalls(iter((f"delete None {file_id}",))), + files=iter((Success(status_code=200, data=FileDeleteResponse(id=file_id)),)), + ) with pytest.raises(AssertionError, match="did not confirm deletion"): cleanup_file(client, file_id, key="test-key") + client.calls.assert_done() @pytest.mark.parametrize("cap", CAPABILITIES, ids=[cap.id for cap in CAPABILITIES]) def test_deletes_raw_files_through_the_upload_provider(self, cap: Capability) -> None: - client: Final = CleanupClient(files=iter((deleted_file(),))) - cleanup_file(client, "file-1", key="test-key", provider=cap.file_provider) expected_provider: Final = cap.provider if cap.scenario in {"model_param", "provider_fallback"} else None - assert client.calls == [f"delete {expected_provider} file-1"] + client: Final = CleanupClient( + calls=ExpectedCalls(iter((f"delete {expected_provider} file-1",))), files=iter((deleted_file(),)) + ) + cleanup_file(client, "file-1", key="test-key", provider=cap.file_provider) + client.calls.assert_done() def test_failed_delete_is_reported_after_remaining_resources_are_cleaned(self) -> None: - client: Final = CleanupClient(files=iter((UnknownApiError(status_code=403, body="secret response"),))) + client: Final = CleanupClient( + calls=ExpectedCalls(iter(("delete azure file-1", "delete key test-key"))), + files=iter((UnknownApiError(status_code=403, body="secret response"),)), + ) manager: Final = ResourceManager(client=client, strict_cleanup=True) key: Final = manager.key() manager.defer(lambda: cleanup_file(client, "file-1", key=key, provider="azure")) with pytest.raises(ExceptionGroup) as caught: manager.teardown() - assert client.calls == ["delete azure file-1", "delete key test-key"] + client.calls.assert_done() assert len(caught.value.exceptions) == 1 assert str(caught.value.exceptions[0]) == "Delete file file-1 failed: HTTP 403" def test_success_response_must_confirm_deletion(self) -> None: - client: Final = CleanupClient(files=iter((deleted_file(deleted=False),))) + client: Final = CleanupClient( + calls=ExpectedCalls(iter(("delete None file-1",))), files=iter((deleted_file(deleted=False),)) + ) with pytest.raises(AssertionError, match="did not confirm deletion"): cleanup_file(client, "file-1", key="test-key") + client.calls.assert_done() def test_cleanup_is_idempotent_when_file_is_already_deleted(self) -> None: - client: Final = CleanupClient(files=iter((UnknownApiError(status_code=404, body="missing"),))) + client: Final = CleanupClient( + calls=ExpectedCalls(iter(("delete azure file-1",))), + files=iter((UnknownApiError(status_code=404, body="missing"),)), + ) cleanup_file(client, "file-1", key="test-key", provider="azure") - assert client.calls == ["delete azure file-1"] + client.calls.assert_done() def test_default_resource_cleanup_keeps_existing_best_effort_behavior(self) -> None: - client: Final = CleanupClient(files=iter((UnknownApiError(status_code=403, body="forbidden"),))) + client: Final = CleanupClient( + calls=ExpectedCalls(iter(("delete None file-1", "delete key test-key"))), + files=iter((UnknownApiError(status_code=403, body="forbidden"),)), + ) manager: Final = ResourceManager(client=client) key: Final = manager.key() manager.defer(lambda: cleanup_file(client, "file-1", key=key)) manager.teardown() - assert client.calls == ["delete None file-1", "delete key test-key"] + client.calls.assert_done() class TestCleanupRetries: @@ -112,40 +143,54 @@ class TestCleanupRetries: ) def test_transient_error_retries_and_returns_success(self, failure: Result[FileDeleteResponse]) -> None: outcomes: Final = iter((failure, deleted_file())) - delays: Final[list[float]] = [] - result: Final[Result[FileDeleteResponse]] = cleanup_result(lambda: next(outcomes), wait=delays.append) + delays: Final = ExpectedCalls(iter((1.0,))) + result: Final[Result[FileDeleteResponse]] = cleanup_result(lambda: next(outcomes), wait=delays) assert isinstance(result, Success) and result.data.deleted - assert delays == [1.0] + delays.assert_done() def test_persistent_error_has_bounded_retries(self) -> None: failure: Final = UnknownApiError(status_code=503, body="unavailable") outcomes: Final[Iterator[Result[FileDeleteResponse]]] = iter((failure,) * (len(CLEANUP_DELAYS) + 1)) - delays: Final[list[float]] = [] - result: Final[Result[FileDeleteResponse]] = cleanup_result(lambda: next(outcomes), wait=delays.append) + delays: Final = ExpectedCalls(iter(CLEANUP_DELAYS)) + result: Final[Result[FileDeleteResponse]] = cleanup_result(lambda: next(outcomes), wait=delays) assert result is failure - assert tuple(delays) == CLEANUP_DELAYS + delays.assert_done() assert next(outcomes, None) is None def test_permanent_error_is_not_retried(self) -> None: failure: Final = UnknownApiError(status_code=403, body="forbidden") outcomes: Final = iter((failure, deleted_file())) - delays: Final[list[float]] = [] - assert cleanup_result(lambda: next(outcomes), wait=delays.append) is failure - assert delays == [] + delays: Final = ExpectedCalls[float](iter(())) + assert cleanup_result(lambda: next(outcomes), wait=delays) is failure + delays.assert_done() assert isinstance(next(outcomes), Success) class TestBatchCancellation: def test_cancelling_batch_is_polled_until_terminal_without_cancelling_again(self) -> None: - client: Final = CleanupClient(batches=iter((batch("cancelling"), batch("cancelling"), batch("cancelled")))) - delays: Final[list[float]] = [] - cleanup_batch(client, MANAGED_BATCH_ID, key="test-key", wait=delays.append) - assert client.calls == [f"retrieve None {MANAGED_BATCH_ID}"] * 3 - assert delays == [10.0] + client: Final = CleanupClient( + calls=ExpectedCalls(iter((f"retrieve None {MANAGED_BATCH_ID}",) * 3)), + batches=iter((batch("cancelling"), batch("cancelling"), batch("cancelled"))), + ) + delays: Final = ExpectedCalls(iter((10.0,))) + cleanup_batch(client, MANAGED_BATCH_ID, key="test-key", wait=delays) + client.calls.assert_done() + delays.assert_done() def test_cancellation_timeout_is_reported_but_file_and_key_cleanup_still_run(self) -> None: client: Final = CleanupClient( - batches=iter((batch("cancelling"), batch("cancelling"))), files=iter((deleted_file(),)) + calls=ExpectedCalls( + iter( + ( + f"retrieve None {MANAGED_BATCH_ID}", + f"retrieve None {MANAGED_BATCH_ID}", + "delete None file-1", + "delete key test-key", + ) + ) + ), + batches=iter((batch("cancelling"), batch("cancelling"))), + files=iter((deleted_file(),)), ) ticks: Final = iter((0.0, BATCH_CANCEL_TIMEOUT_SECONDS)) manager: Final = ResourceManager(client=client, strict_cleanup=True) @@ -155,29 +200,29 @@ class TestBatchCancellation: with pytest.raises(ExceptionGroup) as caught: manager.teardown() assert "cancellation did not finish" in str(caught.value.exceptions[0]) - assert client.calls == [ - f"retrieve None {MANAGED_BATCH_ID}", - f"retrieve None {MANAGED_BATCH_ID}", - "delete None file-1", - "delete key test-key", - ] + client.calls.assert_done() @pytest.mark.parametrize("status", ["completed", "failed", "expired", "cancelled"]) def test_inactive_batch_needs_no_cancellation(self, status: str) -> None: - client: Final = CleanupClient(batches=iter((batch(status),))) + client: Final = CleanupClient( + calls=ExpectedCalls(iter(("retrieve None batch-1",))), batches=iter((batch(status),)) + ) cleanup_batch(client, "batch-1", key="test-key") - assert client.calls == ["retrieve None batch-1"] + client.calls.assert_done() def test_active_batch_is_cancelled_through_its_provider(self) -> None: client: Final = CleanupClient( - batches=iter((batch("in_progress"), batch("cancelled"))), cancellations=iter((batch("cancelling"),)) + calls=ExpectedCalls(iter(("retrieve azure batch-1", "cancel azure batch-1"))), + batches=iter((batch("in_progress"), batch("cancelled"))), + cancellations=iter((batch("cancelling"),)), ) cleanup_batch(client, "batch-1", key="test-key", provider="azure") - assert client.calls == ["retrieve azure batch-1", "cancel azure batch-1"] + client.calls.assert_done() @pytest.mark.parametrize("status", ["completed", "in_progress"]) def test_cancellation_conflict_is_accepted_only_when_batch_became_inactive(self, status: str) -> None: client: Final = CleanupClient( + calls=ExpectedCalls(iter(("retrieve None batch-1", "cancel None batch-1", "retrieve None batch-1"))), batches=iter((batch("in_progress"), batch(status))), cancellations=iter((UnknownApiError(status_code=409, body="conflict"),)), ) @@ -186,7 +231,7 @@ class TestBatchCancellation: else: with pytest.raises(AssertionError, match="Cancel batch batch-1 left status in_progress"): cleanup_batch(client, "batch-1", key="test-key") - assert client.calls == ["retrieve None batch-1", "cancel None batch-1", "retrieve None batch-1"] + client.calls.assert_done() class TestAzureFileExpiry: From c84131b81ab6feb552e23c5996559159e8810066 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 7 Sep 2026 16:00:20 -0700 Subject: [PATCH 063/411] feat(proxy): price cache and reasoning tokens in /cost/estimate POST /cost/estimate now accepts cache_read_input_tokens, cache_creation_input_tokens and reasoning_tokens, bills them at the model's cache and reasoning rates, and reports each share per request, per day and per month next to the rates it used. Custom-priced deployments also get cache and reasoning lines in the cost breakdown now, so the estimate and the spend logs reconcile with their totals instead of showing zero for those tokens. Requested by a customer (Pylon #7365). Claude-Session: https://claude.ai/code/session_011Tn3657NkV6ojLqewL64Kb --- litellm/cost_calculator.py | 1 + .../litellm_core_utils/llm_cost_calc/utils.py | 68 +++-- litellm/proxy/_types.py | 36 +++ .../cost_tracking_settings.py | 253 +++++++++++------- .../llm_cost_calc/test_llm_cost_calc_utils.py | 64 +++++ .../test_cost_tracking_settings.py | 187 ++++++++++++- tests/test_litellm/test_cost_calculator.py | 50 ++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 103 ++++++- 8 files changed, 641 insertions(+), 121 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9a9d2ceda03..a1181ee4124 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1746,6 +1746,7 @@ def completion_cost( service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, + custom_cost_per_token=custom_cost_per_token, ) _reasoning_cost = _token_type_breakdown.reasoning_cost _cache_read_cost = _token_type_breakdown.cache_read_cost diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 68dc27ec25e..8d53c8da3e6 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -19,6 +19,7 @@ from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, CompletionTokensDetailsWrapper, + CostPerToken, DataResidency, ImageResponse, ModelInfo, @@ -1307,6 +1308,40 @@ class TokenTypeCostBreakdown: cache_creation_cost: float +def _reasoning_token_count(usage: Usage) -> int: + parsed: Final = ( + parse_completion_tokens_details(usage)["reasoning_tokens"] if usage.completion_tokens_details is not None else 0 + ) + return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0)) + + +def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]: + """(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details + first, then the private top-level counters the Usage constructor mirrors cache tokens onto for + providers/callers that bypass the details.""" + parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None + parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0 + parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0 + return ( + parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)), + parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)), + parsed["cache_creation_token_details"] if parsed is not None else None, + ) + + +def _custom_pricing_token_type_breakdown(usage: Usage, custom_cost_per_token: CostPerToken) -> TokenTypeCostBreakdown: + """Flat custom pricing has no tiers, uplifts or reasoning rate: cache tokens bill at the configured + cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does.""" + input_rate: Final = custom_cost_per_token["input_cost_per_token"] + cache_read_tokens, cache_creation_tokens, _ = _cache_token_counts(usage) + return TokenTypeCostBreakdown( + reasoning_cost=float(_reasoning_token_count(usage)) * custom_cost_per_token["output_cost_per_token"], + cache_read_cost=float(cache_read_tokens) * custom_cost_per_token.get("cache_read_input_token_cost", input_rate), + cache_creation_cost=float(cache_creation_tokens) + * custom_cost_per_token.get("cache_creation_input_token_cost", input_rate), + ) + + def get_token_type_cost_breakdown( model: str, custom_llm_provider: str | None, @@ -1315,6 +1350,7 @@ def get_token_type_cost_breakdown( data_residency: str | None = None, vertex_location: str | None = None, current_time: datetime | None = None, + custom_cost_per_token: CostPerToken | None = None, ) -> TokenTypeCostBreakdown: """ Provider-agnostic cost of reasoning and cache tokens, derived from the usage @@ -1325,9 +1361,13 @@ def get_token_type_cost_breakdown( land on ``prompt_tokens_details`` (via the Usage constructor and provider transformations) and reasoning tokens on ``completion_tokens_details``. It reuses the same rate-resolution primitives as the total-cost path so the breakdown can - never drift from the totals. Returns zeros (never raises) when the model or its - pricing cannot be resolved. + never drift from the totals. A deployment billed by ``custom_cost_per_token`` is + priced from those flat rates instead of the cost map, for the same reason. + Returns zeros (never raises) when the model or its pricing cannot be resolved. """ + if custom_cost_per_token is not None: + return _custom_pricing_token_type_breakdown(usage=usage, custom_cost_per_token=custom_cost_per_token) + try: model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) except Exception: @@ -1348,12 +1388,6 @@ def get_token_type_cost_breakdown( threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), ) - reasoning_tokens = ( - parse_completion_tokens_details(usage)["reasoning_tokens"] if usage.completion_tokens_details is not None else 0 - ) - if not reasoning_tokens: - reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0)) - reasoning_rate: Final = _resolve_billed_reasoning_rate( model_info=model_info, usage=usage, @@ -1361,23 +1395,9 @@ def get_token_type_cost_breakdown( completion_base_cost=completion_base_cost, current_time=billing_time, ) - reasoning_cost = float(reasoning_tokens) * reasoning_rate - - cache_read_tokens = 0 - cache_creation_tokens = 0 - cache_creation_token_details: CacheCreationTokenDetails | None = None - if usage.prompt_tokens_details is not None: - prompt_tokens_details: Final = parse_prompt_tokens_details(usage) - cache_read_tokens = prompt_tokens_details["cache_hit_tokens"] - cache_creation_tokens = prompt_tokens_details["cache_creation_tokens"] - cache_creation_token_details = prompt_tokens_details["cache_creation_token_details"] - # Fall back to the private top-level counters the Usage constructor mirrors cache - # tokens onto, so providers/callers that bypass prompt_tokens_details are covered. - if not cache_read_tokens: - cache_read_tokens = _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)) - if not cache_creation_tokens: - cache_creation_tokens = _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)) + reasoning_cost = float(_reasoning_token_count(usage)) * reasoning_rate + cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage) cache_read_cost = float(cache_read_tokens) * cache_read_cost_rate cache_creation_cost = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index abce10690e5..65b7d409d7c 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5137,9 +5137,26 @@ class CostEstimateRequest(LiteLLMPydanticObjectBase): model: str = Field(description="Model name (from /model_group/info)") input_tokens: int = Field(description="Expected input tokens per request", ge=0) output_tokens: int = Field(description="Expected output tokens per request", ge=0) + cache_read_input_tokens: int = Field( + default=0, description="Input tokens read from the prompt cache; counted within input_tokens", ge=0 + ) + cache_creation_input_tokens: int = Field( + default=0, description="Input tokens written to the prompt cache; counted within input_tokens", ge=0 + ) + reasoning_tokens: int = Field( + default=0, description="Reasoning tokens the model emits; counted within output_tokens", ge=0 + ) num_requests_per_day: int | None = Field(default=None, description="Number of requests per day", ge=0) num_requests_per_month: int | None = Field(default=None, description="Number of requests per month", ge=0) + @model_validator(mode="after") + def validate_token_subsets(self) -> "CostEstimateRequest": + if self.cache_read_input_tokens + self.cache_creation_input_tokens > self.input_tokens: + raise ValueError("cache_read_input_tokens plus cache_creation_input_tokens cannot exceed input_tokens") + if self.reasoning_tokens > self.output_tokens: + raise ValueError("reasoning_tokens cannot exceed output_tokens") + return self + class CostEstimateResponse(LiteLLMPydanticObjectBase): """Response body for /cost/estimate endpoint.""" @@ -5147,6 +5164,9 @@ class CostEstimateResponse(LiteLLMPydanticObjectBase): model: str input_tokens: int output_tokens: int + cache_read_input_tokens: int = 0 + cache_creation_input_tokens: int = 0 + reasoning_tokens: int = 0 num_requests_per_day: int | None = None num_requests_per_month: int | None = None # Per-request costs @@ -5154,17 +5174,33 @@ class CostEstimateResponse(LiteLLMPydanticObjectBase): input_cost_per_request: float = Field(description="Input token cost per request (before margin)") output_cost_per_request: float = Field(description="Output token cost per request (before margin)") margin_cost_per_request: float = Field(default=0.0, description="Margin/fee added per request") + cache_read_cost_per_request: float = Field(default=0.0, description="Cache-read share of input_cost_per_request") + cache_creation_cost_per_request: float = Field( + default=0.0, description="Cache-write share of input_cost_per_request" + ) + reasoning_cost_per_request: float = Field(default=0.0, description="Reasoning share of output_cost_per_request") # Daily costs (if num_requests_per_day provided) daily_cost: float | None = Field(default=None, description="Total daily cost (includes margin)") daily_input_cost: float | None = Field(default=None, description="Daily input token cost") daily_output_cost: float | None = Field(default=None, description="Daily output token cost") daily_margin_cost: float | None = Field(default=None, description="Daily margin/fee") + daily_cache_read_cost: float | None = Field(default=None, description="Cache-read share of daily_input_cost") + daily_cache_creation_cost: float | None = Field(default=None, description="Cache-write share of daily_input_cost") + daily_reasoning_cost: float | None = Field(default=None, description="Reasoning share of daily_output_cost") # Monthly costs (if num_requests_per_month provided) monthly_cost: float | None = Field(default=None, description="Total monthly cost (includes margin)") monthly_input_cost: float | None = Field(default=None, description="Monthly input token cost") monthly_output_cost: float | None = Field(default=None, description="Monthly output token cost") monthly_margin_cost: float | None = Field(default=None, description="Monthly margin/fee") + monthly_cache_read_cost: float | None = Field(default=None, description="Cache-read share of monthly_input_cost") + monthly_cache_creation_cost: float | None = Field( + default=None, description="Cache-write share of monthly_input_cost" + ) + monthly_reasoning_cost: float | None = Field(default=None, description="Reasoning share of monthly_output_cost") # Pricing info input_cost_per_token: float | None = None output_cost_per_token: float | None = None + cache_read_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-read token") + cache_creation_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-write token") + output_cost_per_reasoning_token: float | None = Field(default=None, description="Rate billed per reasoning token") provider: str | None = None diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 204051c3715..51f31a757cd 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -27,7 +27,15 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.types.utils import CostPerToken, LlmProvidersSet, ModelInfo +from litellm.types.utils import ( + CostBreakdown, + CostPerToken, + LlmProvidersSet, + ModelInfo, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +) router: Final = APIRouter() @@ -46,13 +54,15 @@ def _configured_price(key: str, sources: tuple[Mapping[str, object], ...]) -> fl def _extract_custom_pricing( - litellm_params: Mapping[str, object], model_info: Mapping[str, object] + litellm_params: Mapping[str, object], model_info: Mapping[str, object], builtin: ModelInfo | None ) -> CostPerToken | None: """ Pull per-token pricing configured on a deployment so on-prem / self-hosted models (absent from the public cost map) still estimate a real cost. Pricing may live on ``litellm_params`` or ``model_info``; ``litellm_params`` - wins, matching the router's cost-map registration precedence. + wins, matching the router's cost-map registration precedence. Cache rates the + deployment leaves unset come from the backend model's built-in entry, then its + own input rate, again matching what the router registers for live billing. """ sources: Final = (litellm_params, model_info) input_price: Final = _configured_price("input_cost_per_token", sources) @@ -61,9 +71,15 @@ def _extract_custom_pricing( if input_price is None and output_price is None: return None + input_rate: Final = input_price or 0.0 + cache_sources: Final = sources if builtin is None else (*sources, builtin) + cache_read_price: Final = _configured_price("cache_read_input_token_cost", cache_sources) + cache_creation_price: Final = _configured_price("cache_creation_input_token_cost", cache_sources) return CostPerToken( - input_cost_per_token=input_price or 0.0, + input_cost_per_token=input_rate, output_cost_per_token=output_price or 0.0, + cache_read_input_token_cost=input_rate if cache_read_price is None else cache_read_price, + cache_creation_input_token_cost=input_rate if cache_creation_price is None else cache_creation_price, ) @@ -98,17 +114,14 @@ def _resolve_model_for_cost_lookup(model: str) -> ResolvedCostModel: model_info: Final = first_deployment.get("model_info", {}) custom_llm_provider: Final = litellm_params.get("custom_llm_provider") provider: Final = str(custom_llm_provider) if custom_llm_provider is not None else None - custom_cost_per_token: Final = _extract_custom_pricing(litellm_params, model_info) - - # Check base_model first (needed for Azure custom deployment names) + # base_model wins (needed for Azure custom deployment names) base_model: Final = model_info.get("base_model") or litellm_params.get("base_model") - if base_model: - verbose_proxy_logger.debug("Resolved model '%s' to base_model '%s' from router", model, base_model) - return ResolvedCostModel(str(base_model), provider, custom_cost_per_token) - - resolved_model: Final = litellm_params.get("model") + resolved_model: Final = base_model or litellm_params.get("model") if resolved_model: verbose_proxy_logger.debug("Resolved model '%s' to '%s' from router", model, resolved_model) + custom_cost_per_token: Final = _extract_custom_pricing( + litellm_params, model_info, _lookup_model_info(str(resolved_model)) + ) return ResolvedCostModel(str(resolved_model), provider, custom_cost_per_token) except Exception as e: verbose_proxy_logger.debug("Could not resolve model '%s' from router: %s", model, e) @@ -117,19 +130,97 @@ def _resolve_model_for_cost_lookup(model: str) -> ResolvedCostModel: return ResolvedCostModel(model, None, None) -def _calculate_period_costs(num_requests, cost_per_request, input_cost, output_cost, margin_cost): - """ - Calculate costs for a given number of requests. +@dataclass(frozen=True, slots=True) +class CostLines: + """Cost of one request split the way the spend logs split it: the cache lines are + shares of input_cost and the reasoning line is a share of output_cost.""" - Returns tuple of (total_cost, input_cost, output_cost, margin_cost) or all None if num_requests is None/0. - """ - if not num_requests: - return None, None, None, None - return ( - cost_per_request * num_requests, - input_cost * num_requests, - output_cost * num_requests, - margin_cost * num_requests, + total_cost: float + input_cost: float + output_cost: float + margin_cost: float + cache_read_cost: float + cache_creation_cost: float + reasoning_cost: float + + def times(self, num_requests: int | None) -> "CostLines | None": + if not num_requests: + return None + return CostLines( + total_cost=self.total_cost * num_requests, + input_cost=self.input_cost * num_requests, + output_cost=self.output_cost * num_requests, + margin_cost=self.margin_cost * num_requests, + cache_read_cost=self.cache_read_cost * num_requests, + cache_creation_cost=self.cache_creation_cost * num_requests, + reasoning_cost=self.reasoning_cost * num_requests, + ) + + +def _cost_lines(cost_per_request: float, cost_breakdown: CostBreakdown | None) -> CostLines: + breakdown: Final = cost_breakdown if cost_breakdown is not None else CostBreakdown() + return CostLines( + total_cost=cost_per_request, + input_cost=breakdown.get("input_cost", 0.0), + output_cost=breakdown.get("output_cost", 0.0), + margin_cost=breakdown.get("margin_total_amount", 0.0), + cache_read_cost=breakdown.get("cache_read_cost", 0.0), + cache_creation_cost=breakdown.get("cache_creation_cost", 0.0), + reasoning_cost=breakdown.get("reasoning_cost", 0.0), + ) + + +@dataclass(frozen=True, slots=True) +class EffectiveTokenRates: + input_cost_per_token: float | None + output_cost_per_token: float | None + cache_read_input_token_cost: float | None + cache_creation_input_token_cost: float | None + output_cost_per_reasoning_token: float | None + + +def _custom_token_rates(custom_cost_per_token: CostPerToken) -> EffectiveTokenRates: + input_rate: Final = custom_cost_per_token["input_cost_per_token"] + output_rate: Final = custom_cost_per_token["output_cost_per_token"] + return EffectiveTokenRates( + input_cost_per_token=input_rate, + output_cost_per_token=output_rate, + cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate), + cache_creation_input_token_cost=custom_cost_per_token.get("cache_creation_input_token_cost", input_rate), + output_cost_per_reasoning_token=output_rate, + ) + + +def _cost_map_token_rates(model_info: ModelInfo | None) -> EffectiveTokenRates: + """Base rates the cost calculator bills flat usage at: a cost-map model without a cache price + bills cache tokens at zero, and one without a reasoning price bills reasoning at the output rate.""" + if model_info is None: + return EffectiveTokenRates(None, None, None, None, None) + sources: Final = (model_info,) + output_rate: Final = _configured_price("output_cost_per_token", sources) + reasoning_rate: Final = _configured_price("output_cost_per_reasoning_token", sources) + return EffectiveTokenRates( + input_cost_per_token=_configured_price("input_cost_per_token", sources), + output_cost_per_token=output_rate, + cache_read_input_token_cost=_configured_price("cache_read_input_token_cost", sources) or 0.0, + cache_creation_input_token_cost=_configured_price("cache_creation_input_token_cost", sources) or 0.0, + output_cost_per_reasoning_token=output_rate if reasoning_rate is None else reasoning_rate, + ) + + +def _usage_for_estimate(request: CostEstimateRequest) -> Usage: + cache_tokens: Final = request.cache_read_input_tokens + request.cache_creation_input_tokens + return Usage( + prompt_tokens=request.input_tokens, + completion_tokens=request.output_tokens, + total_tokens=request.input_tokens + request.output_tokens, + reasoning_tokens=request.reasoning_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=request.cache_read_input_tokens, + cache_creation_tokens=request.cache_creation_input_tokens, + ) + if cache_tokens + else None, ) @@ -530,11 +621,14 @@ async def estimate_cost( - model: Model name (e.g., "gpt-4", "claude-3-opus") - input_tokens: Expected input tokens per request - output_tokens: Expected output tokens per request + - cache_read_input_tokens: Cache-read tokens per request, counted within input_tokens (optional) + - cache_creation_input_tokens: Cache-write tokens per request, counted within input_tokens (optional) + - reasoning_tokens: Reasoning tokens per request, counted within output_tokens (optional) - num_requests_per_day: Number of requests per day (optional) - num_requests_per_month: Number of requests per month (optional) Returns cost breakdown including: - - Per-request costs (input, output, margin) + - Per-request costs (input, output, margin, plus the cache-read, cache-write and reasoning shares) - Daily costs (if num_requests_per_day provided) - Monthly costs (if num_requests_per_month provided) @@ -543,14 +637,15 @@ async def estimate_cost( { "model": "gpt-4", "input_tokens": 1000, + "cache_read_input_tokens": 800, "output_tokens": 500, + "reasoning_tokens": 200, "num_requests_per_day": 100, "num_requests_per_month": 3000 } ``` """ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.types.utils import ModelResponse, Usage # Resolve model name (handles router aliases like 'e-model-router' -> 'azure_ai/gpt-4') resolved: Final = _resolve_model_for_cost_lookup(request.model) @@ -559,15 +654,7 @@ async def estimate_cost( verbose_proxy_logger.debug("Cost estimate: request.model='%s' resolved to '%s'", request.model, resolved_model) - # Create a mock response with usage for completion_cost - mock_response: Final = ModelResponse( - model=resolved_model, - usage=Usage( - prompt_tokens=request.input_tokens, - completion_tokens=request.output_tokens, - total_tokens=request.input_tokens + request.output_tokens, - ), - ) + mock_response: Final = ModelResponse(model=resolved_model, usage=_usage_for_estimate(request)) # Create a logging object to capture cost breakdown litellm_logging_obj: Final = LiteLLMLoggingObj( @@ -597,75 +684,53 @@ async def estimate_cost( }, ) - # Get cost breakdown from the logging object - cost_breakdown: Final = litellm_logging_obj.cost_breakdown - - input_cost: Final = cost_breakdown.get("input_cost", 0.0) if cost_breakdown else 0.0 - output_cost: Final = cost_breakdown.get("output_cost", 0.0) if cost_breakdown else 0.0 - margin_cost: Final = cost_breakdown.get("margin_total_amount", 0.0) if cost_breakdown else 0.0 + per_request: Final = _cost_lines(cost_per_request, litellm_logging_obj.cost_breakdown) + daily: Final = per_request.times(request.num_requests_per_day) + monthly: Final = per_request.times(request.num_requests_per_month) model_info: Final = _lookup_model_info(resolved_model) - mapped_input_price: Final = model_info.get("input_cost_per_token") if model_info is not None else None - mapped_output_price: Final = model_info.get("output_cost_per_token") if model_info is not None else None + rates: Final = ( + _custom_token_rates(resolved.custom_cost_per_token) + if resolved.custom_cost_per_token is not None + else _cost_map_token_rates(model_info) + ) mapped_provider: Final = model_info.get("litellm_provider") if model_info is not None else None - - input_cost_per_token: Final = ( - resolved.custom_cost_per_token["input_cost_per_token"] - if resolved.custom_cost_per_token is not None - else mapped_input_price - ) - output_cost_per_token: Final = ( - resolved.custom_cost_per_token["output_cost_per_token"] - if resolved.custom_cost_per_token is not None - else mapped_output_price - ) custom_llm_provider: Final = mapped_provider if mapped_provider is not None else resolved_provider - # Calculate daily and monthly costs - ( - daily_cost, - daily_input_cost, - daily_output_cost, - daily_margin_cost, - ) = _calculate_period_costs( - num_requests=request.num_requests_per_day, - cost_per_request=cost_per_request, - input_cost=input_cost, - output_cost=output_cost, - margin_cost=margin_cost, - ) - ( - monthly_cost, - monthly_input_cost, - monthly_output_cost, - monthly_margin_cost, - ) = _calculate_period_costs( - num_requests=request.num_requests_per_month, - cost_per_request=cost_per_request, - input_cost=input_cost, - output_cost=output_cost, - margin_cost=margin_cost, - ) - return CostEstimateResponse( model=request.model, input_tokens=request.input_tokens, output_tokens=request.output_tokens, + cache_read_input_tokens=request.cache_read_input_tokens, + cache_creation_input_tokens=request.cache_creation_input_tokens, + reasoning_tokens=request.reasoning_tokens, num_requests_per_day=request.num_requests_per_day, num_requests_per_month=request.num_requests_per_month, - cost_per_request=cost_per_request, - input_cost_per_request=input_cost, - output_cost_per_request=output_cost, - margin_cost_per_request=margin_cost, - daily_cost=daily_cost, - daily_input_cost=daily_input_cost, - daily_output_cost=daily_output_cost, - daily_margin_cost=daily_margin_cost, - monthly_cost=monthly_cost, - monthly_input_cost=monthly_input_cost, - monthly_output_cost=monthly_output_cost, - monthly_margin_cost=monthly_margin_cost, - input_cost_per_token=input_cost_per_token, - output_cost_per_token=output_cost_per_token, + cost_per_request=per_request.total_cost, + input_cost_per_request=per_request.input_cost, + output_cost_per_request=per_request.output_cost, + margin_cost_per_request=per_request.margin_cost, + cache_read_cost_per_request=per_request.cache_read_cost, + cache_creation_cost_per_request=per_request.cache_creation_cost, + reasoning_cost_per_request=per_request.reasoning_cost, + daily_cost=daily.total_cost if daily is not None else None, + daily_input_cost=daily.input_cost if daily is not None else None, + daily_output_cost=daily.output_cost if daily is not None else None, + daily_margin_cost=daily.margin_cost if daily is not None else None, + daily_cache_read_cost=daily.cache_read_cost if daily is not None else None, + daily_cache_creation_cost=daily.cache_creation_cost if daily is not None else None, + daily_reasoning_cost=daily.reasoning_cost if daily is not None else None, + monthly_cost=monthly.total_cost if monthly is not None else None, + monthly_input_cost=monthly.input_cost if monthly is not None else None, + monthly_output_cost=monthly.output_cost if monthly is not None else None, + monthly_margin_cost=monthly.margin_cost if monthly is not None else None, + monthly_cache_read_cost=monthly.cache_read_cost if monthly is not None else None, + monthly_cache_creation_cost=monthly.cache_creation_cost if monthly is not None else None, + monthly_reasoning_cost=monthly.reasoning_cost if monthly is not None else None, + input_cost_per_token=rates.input_cost_per_token, + output_cost_per_token=rates.output_cost_per_token, + cache_read_input_token_cost=rates.cache_read_input_token_cost, + cache_creation_input_token_cost=rates.cache_creation_input_token_cost, + output_cost_per_reasoning_token=rates.output_cost_per_reasoning_token, provider=custom_llm_provider, ) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 59f0938e338..7065003839b 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -3874,6 +3874,70 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_co assert text_input_cost + breakdown.cache_read_cost == pytest.approx(prompt_cost) +def _custom_priced_usage() -> Usage: + return Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800, cache_creation_tokens=100), + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200), + ) + + +def test_token_type_cost_breakdown_prices_custom_pricing_from_its_flat_rates(): + """ + A custom-priced deployment, usually absent from the cost map, used to get zero cache and + reasoning lines while its total already billed cache tokens at the custom cache rates. + The lines must come from the same flat rates: a configured cache rate, else the input + rate for cache tokens and the output rate for reasoning tokens. + """ + from litellm.types.utils import CostPerToken + + breakdown = get_token_type_cost_breakdown( + model="openai/onprem-model", + custom_llm_provider="openai", + usage=_custom_priced_usage(), + custom_cost_per_token=CostPerToken( + input_cost_per_token=1e-6, output_cost_per_token=2e-6, cache_read_input_token_cost=1e-7 + ), + ) + + assert breakdown.cache_read_cost == pytest.approx(800 * 1e-7) + assert breakdown.cache_creation_cost == pytest.approx(100 * 1e-6) + assert breakdown.reasoning_cost == pytest.approx(200 * 2e-6) + + +def test_token_type_cost_breakdown_reconciles_with_custom_pricing_totals(): + from litellm.cost_calculator import cost_per_token + from litellm.types.utils import CostPerToken + + usage = _custom_priced_usage() + custom_cost_per_token = CostPerToken( + input_cost_per_token=1e-6, + output_cost_per_token=2e-6, + cache_read_input_token_cost=1e-7, + cache_creation_input_token_cost=1.25e-6, + ) + + prompt_cost, completion_cost = cost_per_token( + model="openai/onprem-model", + custom_llm_provider="openai", + prompt_tokens=1000, + completion_tokens=500, + usage_object=usage, + custom_cost_per_token=custom_cost_per_token, + ) + breakdown = get_token_type_cost_breakdown( + model="openai/onprem-model", + custom_llm_provider="openai", + usage=usage, + custom_cost_per_token=custom_cost_per_token, + ) + + assert 100 * 1e-6 + breakdown.cache_read_cost + breakdown.cache_creation_cost == pytest.approx(prompt_cost) + assert 300 * 2e-6 + breakdown.reasoning_cost == pytest.approx(completion_cost) + + def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost_map): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index ec62cc47018..6d3ef2ffea9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -8,9 +8,11 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient +from pydantic import ValidationError import litellm +from litellm.proxy._types import CostEstimateRequest from litellm.proxy.management_endpoints.cost_tracking_settings import router from litellm.proxy.proxy_server import app @@ -789,13 +791,13 @@ INPUT_TOKENS = 1000 OUTPUT_TOKENS = 500 -def _router_pricing(**pricing: float) -> MagicMock: +def _router_pricing(model: str = AN_UNDERLYING_MODEL, **pricing: float) -> MagicMock: mock_router = MagicMock() mock_router.get_model_list.return_value = [ { "model_name": AN_ALIAS, "litellm_params": { - "model": AN_UNDERLYING_MODEL, + "model": model, "custom_llm_provider": "openai", **pricing, }, @@ -909,3 +911,184 @@ class TestEstimateCostPeriodTotals: assert response.cost_per_request == pytest.approx(0.0022) assert response.daily_margin_cost == pytest.approx(0.02) assert response.daily_cost == pytest.approx(0.22) + + +CACHE_READ_TOKENS = 800 +CACHE_CREATION_TOKENS = 100 +REASONING_TOKENS = 200 +TEXT_INPUT_TOKENS = INPUT_TOKENS - CACHE_READ_TOKENS - CACHE_CREATION_TOKENS +TEXT_OUTPUT_TOKENS = OUTPUT_TOKENS - REASONING_TOKENS + + +async def _estimate_with_cache_and_reasoning(mock_router: MagicMock | None, model: str = AN_ALIAS, **overrides: int): + return await _estimate( + mock_router, + model=model, + cache_read_input_tokens=CACHE_READ_TOKENS, + cache_creation_input_tokens=CACHE_CREATION_TOKENS, + reasoning_tokens=REASONING_TOKENS, + **overrides, + ) + + +class TestEstimateCostCacheAndReasoningTokens: + @pytest.mark.asyncio + async def test_a_mapped_model_bills_cache_and_reasoning_tokens_at_their_own_rates(self, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + A_MAPPED_MODEL, + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "cache_creation_input_token_cost": 3.75e-6, + "output_cost_per_reasoning_token": 1e-5, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + + response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL, num_requests_per_day=10) + + assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 3e-7) + assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 3.75e-6) + assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 1e-5) + assert response.input_cost_per_request == pytest.approx( + TEXT_INPUT_TOKENS * 3e-6 + CACHE_READ_TOKENS * 3e-7 + CACHE_CREATION_TOKENS * 3.75e-6 + ) + assert response.output_cost_per_request == pytest.approx(TEXT_OUTPUT_TOKENS * 15e-6 + REASONING_TOKENS * 1e-5) + assert response.cost_per_request == pytest.approx( + response.input_cost_per_request + response.output_cost_per_request + ) + assert response.daily_cache_read_cost == pytest.approx(10 * CACHE_READ_TOKENS * 3e-7) + assert response.daily_cache_creation_cost == pytest.approx(10 * CACHE_CREATION_TOKENS * 3.75e-6) + assert response.daily_reasoning_cost == pytest.approx(10 * REASONING_TOKENS * 1e-5) + assert response.monthly_cache_read_cost is None + assert response.cache_read_input_token_cost == pytest.approx(3e-7) + assert response.cache_creation_input_token_cost == pytest.approx(3.75e-6) + assert response.output_cost_per_reasoning_token == pytest.approx(1e-5) + assert ( + response.cache_read_input_tokens, + response.cache_creation_input_tokens, + response.reasoning_tokens, + ) == (CACHE_READ_TOKENS, CACHE_CREATION_TOKENS, REASONING_TOKENS) + + @pytest.mark.asyncio + async def test_a_model_without_cache_or_reasoning_prices_estimates_what_the_proxy_bills(self, monkeypatch): + """The cost calculator bills cache tokens of a cost-map model without cache prices at zero + and its reasoning tokens at the output rate. The estimate reports those effective rates.""" + monkeypatch.setitem( + litellm.model_cost, + A_MAPPED_MODEL, + {"input_cost_per_token": 5e-6, "output_cost_per_token": 6e-6, "litellm_provider": "openai", "mode": "chat"}, + ) + + response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL) + + assert response.cache_read_cost_per_request == 0.0 + assert response.cache_creation_cost_per_request == 0.0 + assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 6e-6) + assert response.input_cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6) + assert response.cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6 + OUTPUT_TOKENS * 6e-6) + assert response.cache_read_input_token_cost == 0.0 + assert response.cache_creation_input_token_cost == 0.0 + assert response.output_cost_per_reasoning_token == pytest.approx(6e-6) + + @pytest.mark.asyncio + async def test_a_request_without_cache_or_reasoning_tokens_estimates_as_before(self, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + A_MAPPED_MODEL, + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "cache_creation_input_token_cost": 3.75e-6, + "output_cost_per_reasoning_token": 1e-5, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + + response = await _estimate(None, model=A_MAPPED_MODEL, num_requests_per_day=10) + + assert response.cost_per_request == pytest.approx(INPUT_TOKENS * 3e-6 + OUTPUT_TOKENS * 15e-6) + assert response.cache_read_cost_per_request == 0.0 + assert response.cache_creation_cost_per_request == 0.0 + assert response.reasoning_cost_per_request == 0.0 + assert response.daily_cache_read_cost == 0.0 + assert response.daily_reasoning_cost == 0.0 + + @pytest.mark.asyncio + async def test_a_custom_priced_deployment_bills_cache_and_reasoning_tokens_from_its_flat_rates(self): + response = await _estimate_with_cache_and_reasoning( + _router_pricing(input_cost_per_token=1e-6, output_cost_per_token=2e-6, cache_read_input_token_cost=1e-7) + ) + + assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 1e-7) + assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 1e-6) + assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 2e-6) + assert response.cost_per_request == pytest.approx( + TEXT_INPUT_TOKENS * 1e-6 + CACHE_READ_TOKENS * 1e-7 + CACHE_CREATION_TOKENS * 1e-6 + OUTPUT_TOKENS * 2e-6 + ) + assert response.cache_read_input_token_cost == pytest.approx(1e-7) + assert response.cache_creation_input_token_cost == pytest.approx(1e-6) + assert response.output_cost_per_reasoning_token == pytest.approx(2e-6) + + @pytest.mark.asyncio + async def test_a_custom_priced_deployment_of_a_mapped_model_inherits_its_built_in_cache_rates(self, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + A_MAPPED_MODEL, + { + "input_cost_per_token": 5e-6, + "output_cost_per_token": 6e-6, + "cache_read_input_token_cost": 5e-7, + "cache_creation_input_token_cost": 6.25e-6, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + + response = await _estimate_with_cache_and_reasoning( + _router_pricing(model=A_MAPPED_MODEL, input_cost_per_token=1e-6, output_cost_per_token=2e-6) + ) + + assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 5e-7) + assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 6.25e-6) + assert response.input_cost_per_request == pytest.approx( + TEXT_INPUT_TOKENS * 1e-6 + CACHE_READ_TOKENS * 5e-7 + CACHE_CREATION_TOKENS * 6.25e-6 + ) + assert response.cache_read_input_token_cost == pytest.approx(5e-7) + assert response.cache_creation_input_token_cost == pytest.approx(6.25e-6) + + +class TestCostEstimateRequestTokenSubsets: + def test_cache_tokens_beyond_the_input_tokens_are_rejected(self): + with pytest.raises(ValidationError, match="cannot exceed input_tokens"): + CostEstimateRequest( + model=AN_ALIAS, + input_tokens=INPUT_TOKENS, + output_tokens=OUTPUT_TOKENS, + cache_read_input_tokens=INPUT_TOKENS, + cache_creation_input_tokens=1, + ) + + def test_reasoning_tokens_beyond_the_output_tokens_are_rejected(self): + with pytest.raises(ValidationError, match="cannot exceed output_tokens"): + CostEstimateRequest( + model=AN_ALIAS, + input_tokens=INPUT_TOKENS, + output_tokens=OUTPUT_TOKENS, + reasoning_tokens=OUTPUT_TOKENS + 1, + ) + + def test_the_endpoint_answers_422_when_cache_tokens_exceed_input_tokens(self): + response = client.post( + "/cost/estimate", + headers={"Authorization": "Bearer sk-1234"}, + json={"model": AN_ALIAS, "input_tokens": 1000, "output_tokens": 100, "cache_read_input_tokens": 8000}, + ) + + assert response.status_code == 422 + assert "cannot exceed input_tokens" in response.text diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index f8fa2231597..910587d3dc5 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3736,6 +3736,56 @@ def test_completion_cost_logs_reasoning_and_cache_breakdown(_local_model_cost_ma assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(100 * 3e-08) +def test_completion_cost_logs_cache_and_reasoning_breakdown_for_custom_pricing(): + """ + A custom-priced deployment bills cache tokens at its custom cache rates, but the + breakdown stored for the spend logs carried no cache or reasoning lines for it. + """ + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import CompletionTokensDetailsWrapper, CostPerToken + + logging_obj = Logging( + model="openai/onprem-model", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="custom-pricing-breakdown", + function_id="f", + ) + response = ModelResponse( + model="openai/onprem-model", + usage=Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800, cache_creation_tokens=100), + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200), + ), + ) + + total = completion_cost( + completion_response=response, + model="openai/onprem-model", + custom_llm_provider="openai", + custom_cost_per_token=CostPerToken( + input_cost_per_token=1e-6, + output_cost_per_token=2e-6, + cache_read_input_token_cost=1e-7, + cache_creation_input_token_cost=1.25e-6, + ), + litellm_logging_obj=logging_obj, + ) + + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(800 * 1e-7) + assert logging_obj.cost_breakdown["cache_creation_cost"] == pytest.approx(100 * 1.25e-6) + assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(200 * 2e-6) + assert total == pytest.approx(100 * 1e-6 + 800 * 1e-7 + 100 * 1.25e-6 + 500 * 2e-6) + + def test_cost_per_token_per_second_pricing(monkeypatch): """ Models priced by duration (input/output_cost_per_second) with no per-token rates diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6fb08445aff..6a9c86cc1b5 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -3301,11 +3301,14 @@ export interface paths { * - model: Model name (e.g., "gpt-4", "claude-3-opus") * - input_tokens: Expected input tokens per request * - output_tokens: Expected output tokens per request + * - cache_read_input_tokens: Cache-read tokens per request, counted within input_tokens (optional) + * - cache_creation_input_tokens: Cache-write tokens per request, counted within input_tokens (optional) + * - reasoning_tokens: Reasoning tokens per request, counted within output_tokens (optional) * - num_requests_per_day: Number of requests per day (optional) * - num_requests_per_month: Number of requests per month (optional) * * Returns cost breakdown including: - * - Per-request costs (input, output, margin) + * - Per-request costs (input, output, margin, plus the cache-read, cache-write and reasoning shares) * - Daily costs (if num_requests_per_day provided) * - Monthly costs (if num_requests_per_month provided) * @@ -3314,7 +3317,9 @@ export interface paths { * { * "model": "gpt-4", * "input_tokens": 1000, + * "cache_read_input_tokens": 800, * "output_tokens": 500, + * "reasoning_tokens": 200, * "num_requests_per_day": 100, * "num_requests_per_month": 3000 * } @@ -26392,6 +26397,18 @@ export interface components { * @description Request body for /cost/estimate endpoint. */ CostEstimateRequest: { + /** + * Cache Creation Input Tokens + * @description Input tokens written to the prompt cache; counted within input_tokens + * @default 0 + */ + cache_creation_input_tokens: number; + /** + * Cache Read Input Tokens + * @description Input tokens read from the prompt cache; counted within input_tokens + * @default 0 + */ + cache_read_input_tokens: number; /** * Input Tokens * @description Expected input tokens per request @@ -26417,17 +26434,65 @@ export interface components { * @description Expected output tokens per request */ output_tokens: number; + /** + * Reasoning Tokens + * @description Reasoning tokens the model emits; counted within output_tokens + * @default 0 + */ + reasoning_tokens: number; }; /** * CostEstimateResponse * @description Response body for /cost/estimate endpoint. */ CostEstimateResponse: { + /** + * Cache Creation Cost Per Request + * @description Cache-write share of input_cost_per_request + * @default 0 + */ + cache_creation_cost_per_request: number; + /** + * Cache Creation Input Token Cost + * @description Rate billed per cache-write token + */ + cache_creation_input_token_cost?: number | null; + /** + * Cache Creation Input Tokens + * @default 0 + */ + cache_creation_input_tokens: number; + /** + * Cache Read Cost Per Request + * @description Cache-read share of input_cost_per_request + * @default 0 + */ + cache_read_cost_per_request: number; + /** + * Cache Read Input Token Cost + * @description Rate billed per cache-read token + */ + cache_read_input_token_cost?: number | null; + /** + * Cache Read Input Tokens + * @default 0 + */ + cache_read_input_tokens: number; /** * Cost Per Request * @description Total cost per request (includes margin) */ cost_per_request: number; + /** + * Daily Cache Creation Cost + * @description Cache-write share of daily_input_cost + */ + daily_cache_creation_cost?: number | null; + /** + * Daily Cache Read Cost + * @description Cache-read share of daily_input_cost + */ + daily_cache_read_cost?: number | null; /** * Daily Cost * @description Total daily cost (includes margin) @@ -26448,6 +26513,11 @@ export interface components { * @description Daily output token cost */ daily_output_cost?: number | null; + /** + * Daily Reasoning Cost + * @description Reasoning share of daily_output_cost + */ + daily_reasoning_cost?: number | null; /** * Input Cost Per Request * @description Input token cost per request (before margin) @@ -26465,6 +26535,16 @@ export interface components { margin_cost_per_request: number; /** Model */ model: string; + /** + * Monthly Cache Creation Cost + * @description Cache-write share of monthly_input_cost + */ + monthly_cache_creation_cost?: number | null; + /** + * Monthly Cache Read Cost + * @description Cache-read share of monthly_input_cost + */ + monthly_cache_read_cost?: number | null; /** * Monthly Cost * @description Total monthly cost (includes margin) @@ -26485,10 +26565,20 @@ export interface components { * @description Monthly output token cost */ monthly_output_cost?: number | null; + /** + * Monthly Reasoning Cost + * @description Reasoning share of monthly_output_cost + */ + monthly_reasoning_cost?: number | null; /** Num Requests Per Day */ num_requests_per_day?: number | null; /** Num Requests Per Month */ num_requests_per_month?: number | null; + /** + * Output Cost Per Reasoning Token + * @description Rate billed per reasoning token + */ + output_cost_per_reasoning_token?: number | null; /** * Output Cost Per Request * @description Output token cost per request (before margin) @@ -26500,6 +26590,17 @@ export interface components { output_tokens: number; /** Provider */ provider?: string | null; + /** + * Reasoning Cost Per Request + * @description Reasoning share of output_cost_per_request + * @default 0 + */ + reasoning_cost_per_request: number; + /** + * Reasoning Tokens + * @default 0 + */ + reasoning_tokens: number; }; /** CreateCredentialItem */ CreateCredentialItem: { From 7bff9bf9a2594d3cd5dd6a80a44e4e3b1cd38f61 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 7 Sep 2026 16:22:32 -0700 Subject: [PATCH 064/411] fix(batches): handle provider cancellation and file cleanup gaps --- litellm/llms/bedrock/files/transformation.py | 70 ++++++++------ tests/e2e/batches/COVERAGE.md | 7 +- tests/e2e/batches/batch_cleanup.py | 63 +++++++++--- tests/e2e/batches/test_batch_cleanup.py | 69 +++++++++++++- tests/e2e/batches/test_batches_e2e.py | 14 +-- .../test_bedrock_files_transformation.py | 95 ++++++++++++++++--- 6 files changed, 258 insertions(+), 60 deletions(-) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 33b27943ad8..90b539ff37c 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -7,7 +7,7 @@ from contextlib import suppress from functools import cache from itertools import chain from types import MappingProxyType -from typing import Any, Final, TypeAlias, TypedDict +from typing import Any, Final, Literal, TypeAlias, TypedDict from urllib.parse import unquote import httpx @@ -60,11 +60,8 @@ from litellm.utils import get_llm_provider from ..base_aws_llm import BaseAWSLLM from ..common_utils import BedrockError, merge_bedrock_aws_request_params, resolve_s3_encryption_key_id -# litellm_params key used to hand the SigV4-signed GET headers from -# `transform_file_content_request` to `validate_environment` (the only hook -# the shared file-content HTTP handler exposes for setting request headers). -# Same pattern as the `upload_url` handoff in `transform_create_file_request`. -S3_SIGNED_GET_HEADERS_PARAM: Final = "_s3_signed_get_headers" +S3_SIGNED_REQUEST_HEADERS_PARAM: Final = "_s3_signed_request_headers" +S3_DELETE_FILE_ID_PARAM: Final = "_s3_delete_file_id" # litellm_params key carrying the size of the body uploaded to S3, handed from # `transform_create_file_request` to `transform_create_file_response`. @@ -291,7 +288,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) -> dict: result: Final[dict[str, object]] = {} result.update(headers) - signed_headers: Final = litellm_params.pop(S3_SIGNED_GET_HEADERS_PARAM, None) + signed_headers: Final = litellm_params.pop(S3_SIGNED_REQUEST_HEADERS_PARAM, None) if isinstance(signed_headers, Mapping): result.update(signed_headers) # any-ok: untyped handoff headers # otherwise no extra headers - AWS credentials are handled by BaseAWSLLM @@ -1187,18 +1184,31 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): def transform_delete_file_request( self, file_id: str, - optional_params: dict, - litellm_params: dict, - ) -> tuple[str, dict]: - raise NotImplementedError("BedrockFilesConfig does not support file deletion") + optional_params: Mapping[str, object], + litellm_params: MutableMapping[str, object], + ) -> tuple[str, dict[str, str]]: + request: Final = self._transform_s3_file_request( + file_id=file_id, method="DELETE", optional_params=optional_params, litellm_params=litellm_params + ) + litellm_params[S3_DELETE_FILE_ID_PARAM] = file_id + return request def transform_delete_file_response( self, raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> FileDeleted: - raise NotImplementedError("BedrockFilesConfig does not support file deletion") + if raw_response.status_code != 204: + raise BedrockError( + status_code=raw_response.status_code if raw_response.status_code >= 400 else 502, + message=raw_response.text or f"S3 file deletion returned HTTP {raw_response.status_code}", + headers=raw_response.headers, + ) + file_id: Final = litellm_params.get(S3_DELETE_FILE_ID_PARAM) + if not isinstance(file_id, str) or not file_id: + raise ValueError("Missing file id for Bedrock file deletion response") + return FileDeleted(id=file_id, deleted=True, object="file") def transform_list_files_request( self, @@ -1233,6 +1243,18 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): if not file_id: raise ValueError("file_id is required for Bedrock file content retrieval") + return self._transform_s3_file_request( + file_id=file_id, method="GET", optional_params=optional_params, litellm_params=litellm_params + ) + + def _transform_s3_file_request( + self, + *, + file_id: str, + method: Literal["GET", "DELETE"], + optional_params: Mapping[str, object], + litellm_params: MutableMapping[str, object], + ) -> tuple[str, dict[str, str]]: s3_uri: Final = extract_s3_uri_from_file_id(file_id) bucket_name, object_key = _validate_file_id_against_configured_buckets( s3_uri=s3_uri, @@ -1240,40 +1262,32 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): allow_legacy_cloud_file_ids=should_allow_legacy_cloud_file_ids(litellm_params), ) - # The shared file-content handler passes optional_params={}, so AWS - # credentials/region arrive via litellm_params here (unlike the upload - # path). s3_region_name wins over aws_region_name, same priority as - # get_complete_file_url above. - merged_params: Final[dict[str, object]] = {} - merged_params.update(litellm_params) - merged_params.update(optional_params) - request_params: Final = _BedrockS3RequestParams.model_validate(merged_params) + request_params: Final = _BedrockS3RequestParams.model_validate({**litellm_params, **optional_params}) region_preference: Final = request_params.s3_region_name or request_params.aws_region_name region_params: Final[dict[str, str | None]] = {"aws_region_name": region_preference} aws_region_name: Final = self._get_aws_region_name(optional_params=region_params, model="") - s3_endpoint_url = ( + s3_endpoint_url: Final = ( request_params.s3_endpoint_url or f"https://s3.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}" ).rstrip("/") url: Final = f"{s3_endpoint_url}/{bucket_name}/{encode_s3_object_key_for_url(object_key)}" - litellm_params[S3_SIGNED_GET_HEADERS_PARAM] = self._sign_s3_get_request( + litellm_params[S3_SIGNED_REQUEST_HEADERS_PARAM] = self._sign_s3_request_without_body( api_base=url, aws_region_name=aws_region_name, request_params=request_params, + method=method, ) return url, {} - def _sign_s3_get_request( + def _sign_s3_request_without_body( self, api_base: str, aws_region_name: str, request_params: _BedrockS3RequestParams, + method: Literal["GET", "DELETE"] = "GET", ) -> dict[str, str]: - """ - SigV4-sign an S3 GetObject request, mirroring `_sign_s3_request` (PUT). - """ try: import hashlib @@ -1297,7 +1311,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): empty_body_hash: Final = hashlib.sha256(b"").hexdigest() aws_request: Final = AWSRequest( # any-ok: botocore AWSRequest is untyped - method="GET", + method=method, url=api_base, headers={"x-amz-content-sha256": empty_body_hash}, ) diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index f95ea1f2649..ca44fc95e25 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -131,7 +131,12 @@ failures up to three times. Teardown attempts every registered cleanup before reporting failures as test errors. Already deleted files and batches that are terminal are safe to clean up again. Managed batch cancellation polls for up to eleven minutes before input deletion: the ten-minute provider window plus a propagation margin. -Raw and model-encoded inputs can be deleted after cancellation is accepted +Accepted cancellation may still report validating or in_progress while the provider +updates its state. Raw and model-encoded batches are polled until cancelling or +terminal before input deletion. OpenAI and Azure lifecycle cleanup also deletes +output and error files returned by terminal batches. Bedrock deletion uses a signed S3 DELETE +restricted to the configured storage buckets and managed file prefixes. The low-RPM +test submits with its restricted key and cleans up with the test administrator key Azure input uploads request `expires_after` anchored to `created_at` with `seconds=1209600`, and the lifecycle tests check the returned expiry. This is a diff --git a/tests/e2e/batches/batch_cleanup.py b/tests/e2e/batches/batch_cleanup.py index 722df0c29bc..9284882ad82 100644 --- a/tests/e2e/batches/batch_cleanup.py +++ b/tests/e2e/batches/batch_cleanup.py @@ -1,16 +1,17 @@ +from builtins import ExceptionGroup from collections.abc import Callable from itertools import count from time import monotonic, sleep from typing import Final, Protocol -from pydantic import BaseModel - from batch_client import BatchObject, FileDeleteResponse from capabilities import is_managed_id from e2e_http import NetworkError, RateLimitedError, Result, Success, UnknownApiError +from pydantic import BaseModel CLEANUP_DELAYS: Final = (1.0, 2.0, 4.0) BATCH_TERMINAL_STATUSES: Final = frozenset({"completed", "failed", "expired", "cancelled"}) +BATCH_PENDING_STATUSES: Final = frozenset({"validating", "in_progress", "finalizing", "cancelling"}) BATCH_CANCEL_TIMEOUT_SECONDS: Final = 660.0 BATCH_CANCEL_POLL_SECONDS: Final = 10.0 @@ -63,6 +64,7 @@ def cleanup_batch( *, key: str, provider: str | None = None, + delete_output_files: bool = False, wait: Callable[[float], None] = sleep, clock: Callable[[], float] = monotonic, ) -> None: @@ -72,18 +74,28 @@ def cleanup_batch( f"Retrieve batch {batch_id} for cleanup", ) if fetched.status in BATCH_TERMINAL_STATUSES: + if delete_output_files: + _cleanup_batch_outputs(client, fetched, key=key, provider=provider) return if fetched.status == "cancelling" and not needs_terminal_state: return - if fetched.status != "cancelling": - result: Final = cleanup_result(lambda: client.cancel_batch(batch_id, key=key, provider=provider)) - if not (isinstance(result, UnknownApiError) and result.status_code in {400, 409}): - cancelled: Final = _require_cleanup_success(result, f"Cancel batch {batch_id}") - assert cancelled.status in BATCH_TERMINAL_STATUSES | {"cancelling"}, ( - f"Cancel batch {batch_id} left status {cancelled.status}" - ) - if not needs_terminal_state: - return + result: Final = ( + Success(status_code=200, data=fetched) + if fetched.status == "cancelling" + else cleanup_result(lambda: client.cancel_batch(batch_id, key=key, provider=provider)) + ) + conflicted: Final = isinstance(result, UnknownApiError) and result.status_code in {400, 409} + if not conflicted: + cancelled: Final = _require_cleanup_success(result, f"Cancel batch {batch_id}") + assert cancelled.status in BATCH_TERMINAL_STATUSES | BATCH_PENDING_STATUSES, ( + f"Cancel batch {batch_id} left status {cancelled.status}" + ) + if cancelled.status in BATCH_TERMINAL_STATUSES: + if delete_output_files: + _cleanup_batch_outputs(client, cancelled, key=key, provider=provider) + return + if cancelled.status == "cancelling" and not needs_terminal_state: + return deadline: Final = clock() + BATCH_CANCEL_TIMEOUT_SECONDS for current in ( _require_cleanup_success( @@ -93,11 +105,36 @@ def cleanup_batch( for _ in count() ): if current.status in BATCH_TERMINAL_STATUSES: + if delete_output_files: + _cleanup_batch_outputs(client, current, key=key, provider=provider) return - assert current.status == "cancelling", f"Cancel batch {batch_id} left status {current.status}" - if not needs_terminal_state: + assert current.status in ({"cancelling"} if conflicted else BATCH_PENDING_STATUSES), ( + f"Cancel batch {batch_id} left status {current.status}" + ) + if current.status == "cancelling" and not needs_terminal_state: return assert clock() < deadline, ( f"Batch {batch_id} cancellation did not finish within {BATCH_CANCEL_TIMEOUT_SECONDS}s" ) wait(BATCH_CANCEL_POLL_SECONDS) + + +def _cleanup_batch_outputs(client: BatchCleanupClient, batch: BatchObject, *, key: str, provider: str | None) -> None: + errors: Final = tuple( + error + for file_id in dict.fromkeys((batch.output_file_id, batch.error_file_id)) + if file_id is not None and file_id != batch.input_file_id + if (error := _output_cleanup_error(client, file_id, key=key, provider=provider)) is not None + ) + if errors: + raise ExceptionGroup(f"Batch {batch.id} output cleanup failed", errors) + + +def _output_cleanup_error( + client: BatchCleanupClient, file_id: str, *, key: str, provider: str | None +) -> Exception | None: + try: + cleanup_file(client, file_id, key=key, provider=provider) + except Exception as error: + return error + return None diff --git a/tests/e2e/batches/test_batch_cleanup.py b/tests/e2e/batches/test_batch_cleanup.py index 66b7079ccc2..a875aee719b 100644 --- a/tests/e2e/batches/test_batch_cleanup.py +++ b/tests/e2e/batches/test_batch_cleanup.py @@ -4,7 +4,6 @@ from dataclasses import dataclass, field from typing import Final import pytest - from batch_cleanup import BATCH_CANCEL_TIMEOUT_SECONDS, CLEANUP_DELAYS, cleanup_batch, cleanup_file, cleanup_result from batch_client import AZURE_FILE_EXPIRY_SECONDS, BatchObject, FileDeleteResponse, batch_upload_form from capabilities import CAPABILITIES, Capability @@ -219,6 +218,74 @@ class TestBatchCancellation: cleanup_batch(client, "batch-1", key="test-key", provider="azure") client.calls.assert_done() + @pytest.mark.parametrize("batch_id", ["batch-1", MANAGED_BATCH_ID]) + @pytest.mark.parametrize("pending_status", ["validating", "in_progress"]) + def test_accepted_cancellation_waits_through_stale_provider_status( + self, batch_id: str, pending_status: str + ) -> None: + client: Final = CleanupClient( + calls=ExpectedCalls( + iter( + ( + f"retrieve vertex_ai {batch_id}", + f"cancel vertex_ai {batch_id}", + f"retrieve vertex_ai {batch_id}", + f"retrieve vertex_ai {batch_id}", + f"retrieve vertex_ai {batch_id}", + "delete vertex_ai file-1", + "delete key test-key", + ) + ) + ), + batches=iter((batch("validating"), batch(pending_status), batch(pending_status), batch("cancelled"))), + cancellations=iter((batch(pending_status),)), + files=iter((deleted_file(),)), + ) + delays: Final = ExpectedCalls(iter((10.0, 10.0))) + manager: Final = ResourceManager(client=client, strict_cleanup=True) + key: Final = manager.key() + manager.defer(lambda: cleanup_file(client, "file-1", key=key, provider="vertex_ai")) + manager.defer(lambda: cleanup_batch(client, batch_id, key=key, provider="vertex_ai", wait=delays)) + manager.teardown() + client.calls.assert_done() + delays.assert_done() + + @pytest.mark.parametrize("output_delete_fails", [False, True]) + def test_batch_that_completed_before_cleanup_deletes_output_and_error_files( + self, output_delete_fails: bool + ) -> None: + client: Final = CleanupClient( + calls=ExpectedCalls( + iter(("retrieve openai batch-1", "delete openai file-output", "delete openai file-error")) + ), + batches=iter( + ( + Success( + status_code=200, + data=BatchObject( + id="batch-1", + status="completed", + input_file_id="file-input", + output_file_id="file-output", + error_file_id="file-error", + ), + ), + ) + ), + files=iter( + ( + UnknownApiError(status_code=403, body="forbidden") if output_delete_fails else deleted_file(), + deleted_file(), + ) + ), + ) + if output_delete_fails: + with pytest.raises(ExceptionGroup, match="output cleanup failed"): + cleanup_batch(client, "batch-1", key="test-key", provider="openai", delete_output_files=True) + else: + cleanup_batch(client, "batch-1", key="test-key", provider="openai", delete_output_files=True) + client.calls.assert_done() + @pytest.mark.parametrize("status", ["completed", "in_progress"]) def test_cancellation_conflict_is_accepted_only_when_batch_became_inactive(self, status: str) -> None: client: Final = CleanupClient( diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 1b4a6ed266f..c4b699190b8 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -25,7 +25,7 @@ from datetime import datetime, timedelta, timezone import pytest from pydantic import BaseModel -from e2e_config import PROXY_BASE_URL, unique_marker +from e2e_config import MASTER_KEY, PROXY_BASE_URL, unique_marker from batch_cleanup import cleanup_batch, cleanup_file from batch_client import ( @@ -257,7 +257,9 @@ def test_batch_lifecycle( require_successful_call(created) batch = BatchObject.model_validate_json(created.body) resources.defer( - lambda: cleanup_batch(client, batch.id, key=key, provider=provider) + lambda: cleanup_batch( + client, batch.id, key=key, provider=provider, delete_output_files=cap.provider in {"openai", "azure"} + ) ) assert batch.id, f"create returned no batch id (body={created.body[:200]})" @@ -801,7 +803,7 @@ class TestBatchEnqueuedTokenLimit: """ def _upload_batch_file( - self, client: BatchClient, resources: ResourceManager, key: str + self, client: BatchClient, resources: ResourceManager, key: str, *, cleanup_key: str | None = None ) -> FileObject: file = unwrap( client.upload_file( @@ -811,7 +813,7 @@ class TestBatchEnqueuedTokenLimit: key=key, ) ) - resources.defer(lambda: cleanup_file(client, file.id, key=key)) + resources.defer(lambda: cleanup_file(client, file.id, key=cleanup_key or key)) return file def _generate_enqueued_key( @@ -848,7 +850,7 @@ class TestBatchEnqueuedTokenLimit: marker="rpm", rpm_limit=BATCH_RL_RPM_LIMIT, ) - file = self._upload_batch_file(client, resources, key) + file = self._upload_batch_file(client, resources, key, cleanup_key=MASTER_KEY) created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) @@ -859,7 +861,7 @@ class TestBatchEnqueuedTokenLimit: ) require_successful_call(created) batch = BatchObject.model_validate_json(created.body) - resources.defer(lambda: cleanup_batch(client, batch.id, key=key)) + resources.defer(lambda: cleanup_batch(client, batch.id, key=MASTER_KEY, delete_output_files=True)) @pytest.mark.covers( "quota_management.ratelimit.batch_enqueued_tokens.blocks_when_exhausted", diff --git a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py index 541c0db15d8..2c02a58663e 100644 --- a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py @@ -5,6 +5,8 @@ Test bedrock files transformation functionality import json import os from collections.abc import Mapping +from contextlib import AsyncExitStack, closing +from typing import Final from unittest.mock import MagicMock from urllib.parse import unquote, urlparse @@ -1855,6 +1857,77 @@ class TestBedrockBatchNonChatEndpointRecords: ] +class TestBedrockFileDeletion: + S3_URI: Final = "s3://my-bucket/litellm-bedrock-files-model-abc.jsonl" + URL: Final = "https://s3.us-west-2.amazonaws.com/my-bucket/litellm-bedrock-files-model-abc.jsonl" + + def test_delete_file_sends_signed_delete_and_returns_matching_id(self, monkeypatch: pytest.MonkeyPatch) -> None: + import httpx + import respx + + import litellm + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + monkeypatch.setenv("AWS_S3_BUCKET_NAME", "my-bucket") + with respx.mock, closing(HTTPHandler()) as client: + route: Final = respx.delete(self.URL).mock(return_value=httpx.Response(204)) + deleted: Final = litellm.file_delete( + file_id=self.S3_URI, custom_llm_provider="bedrock", client=client, + aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="test-secret", aws_region_name="us-west-2", + ) + assert route.call_count == 1 + request: Final = route.calls[0].request + assert request.content == b"" + signed: Final = AWSRequest(method="DELETE", url=self.URL, headers={ + "X-Amz-Date": request.headers["X-Amz-Date"], + "X-Amz-Content-SHA256": request.headers["X-Amz-Content-SHA256"], + }) + signed.context["timestamp"] = request.headers["X-Amz-Date"] + auth: Final = S3SigV4Auth(Credentials("AKIAEXAMPLE", "test-secret"), "s3", "us-west-2") + signature: Final = auth.signature(auth.string_to_sign(signed, auth.canonical_request(signed)), signed) + assert request.headers["Authorization"].endswith(f"Signature={signature}") + assert deleted.id == self.S3_URI and deleted.deleted is True + + @pytest.mark.asyncio + async def test_adelete_file_propagates_s3_errors(self, monkeypatch: pytest.MonkeyPatch) -> None: + import httpx + import respx + + import litellm + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + monkeypatch.setenv("AWS_S3_BUCKET_NAME", "my-bucket") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + async with AsyncExitStack() as stack: + client: Final = AsyncHTTPHandler() + stack.push_async_callback(client.close) + with respx.mock: + route: Final = respx.delete(self.URL).mock( + return_value=httpx.Response(403, content=b"AccessDenied") + ) + from litellm.llms.bedrock.common_utils import BedrockError + + with pytest.raises(BedrockError, match="AccessDenied"): + await litellm.afile_delete( + file_id=self.S3_URI, custom_llm_provider="bedrock", client=client, + aws_access_key_id="AKIAEXAMPLE", aws_secret_access_key="test-secret", aws_region_name="us-west-2", + ) + assert route.call_count == 1 + + @pytest.mark.parametrize("file_id, message", [ + ("s3://other-bucket/litellm-bedrock-files-model-abc.jsonl", "configured storage bucket"), + ("s3://my-bucket/private/data.jsonl", "LiteLLM-managed"), + ]) + def test_delete_rejects_untrusted_objects_before_signing( + self, file_id: str, message: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + monkeypatch.setenv("AWS_S3_BUCKET_NAME", "my-bucket") + with pytest.raises(ValueError, match=message): + BedrockFilesConfig().transform_delete_file_request(file_id=file_id, optional_params={}, litellm_params={}) + + class TestBedrockFileContentTransformation: """SigV4-signed S3 GetObject retrieval of Bedrock batch output files.""" @@ -1873,7 +1946,7 @@ class TestBedrockFileContentTransformation: import hashlib from litellm.llms.bedrock.files.transformation import ( - S3_SIGNED_GET_HEADERS_PARAM, + S3_SIGNED_REQUEST_HEADERS_PARAM, BedrockFilesConfig, ) @@ -1889,7 +1962,7 @@ class TestBedrockFileContentTransformation: assert url == self.EXPECTED_URL assert params == {} - signed_headers = litellm_params[S3_SIGNED_GET_HEADERS_PARAM] + signed_headers = litellm_params[S3_SIGNED_REQUEST_HEADERS_PARAM] content_hashes = { value for name, value in signed_headers.items() @@ -2139,7 +2212,7 @@ class TestBedrockFileContentTransformation: def test_s3_region_name_wins_for_content_signing(self, monkeypatch): """s3_region_name must override aws_region_name for both the URL and the signature.""" from litellm.llms.bedrock.files.transformation import ( - S3_SIGNED_GET_HEADERS_PARAM, + S3_SIGNED_REQUEST_HEADERS_PARAM, BedrockFilesConfig, ) @@ -2154,17 +2227,17 @@ class TestBedrockFileContentTransformation: ) assert url.startswith("https://s3.eu-west-1.amazonaws.com/") - authorization = litellm_params[S3_SIGNED_GET_HEADERS_PARAM]["Authorization"] + authorization = litellm_params[S3_SIGNED_REQUEST_HEADERS_PARAM]["Authorization"] assert "/eu-west-1/s3/aws4_request" in authorization def test_validate_environment_merges_and_pops_signed_get_headers(self): from litellm.llms.bedrock.files.transformation import ( - S3_SIGNED_GET_HEADERS_PARAM, + S3_SIGNED_REQUEST_HEADERS_PARAM, BedrockFilesConfig, ) litellm_params = { - S3_SIGNED_GET_HEADERS_PARAM: {"Authorization": "AWS4-HMAC-SHA256 test"} + S3_SIGNED_REQUEST_HEADERS_PARAM: {"Authorization": "AWS4-HMAC-SHA256 test"} } headers = BedrockFilesConfig().validate_environment( @@ -2179,7 +2252,7 @@ class TestBedrockFileContentTransformation: "x-custom": "kept", "Authorization": "AWS4-HMAC-SHA256 test", } - assert S3_SIGNED_GET_HEADERS_PARAM not in litellm_params + assert S3_SIGNED_REQUEST_HEADERS_PARAM not in litellm_params def test_transform_file_content_response_wraps_binary_content(self): import httpx @@ -2379,7 +2452,7 @@ class TestBedrockFilesS3SignatureEncoding: self, monkeypatch: pytest.MonkeyPatch ) -> None: from litellm.llms.bedrock.files.transformation import ( - S3_SIGNED_GET_HEADERS_PARAM, + S3_SIGNED_REQUEST_HEADERS_PARAM, BedrockFilesConfig, ) @@ -2402,7 +2475,7 @@ class TestBedrockFilesS3SignatureEncoding: method="GET", url=url, body=None, - headers=litellm_params[S3_SIGNED_GET_HEADERS_PARAM], + headers=litellm_params[S3_SIGNED_REQUEST_HEADERS_PARAM], ) @@ -2457,7 +2530,7 @@ def test_sign_s3_request_assumes_role_with_external_id(monkeypatch): assert "ASIAFILESPUTROLE" in authorization -def test_sign_s3_get_request_assumes_role_with_external_id(monkeypatch): +def test_sign_s3_request_without_body_assumes_role_with_external_id(monkeypatch): """A trust policy requiring sts:ExternalId must be satisfied when signing the S3 download request.""" import datetime from unittest.mock import patch @@ -2504,7 +2577,7 @@ def test_sign_s3_get_request_assumes_role_with_external_id(monkeypatch): assert request_params.aws_external_id == "external-id-files-get" with patch.object(boto3, "client", return_value=FakeSTSClient()): - signed_headers = BedrockFilesConfig()._sign_s3_get_request( + signed_headers = BedrockFilesConfig()._sign_s3_request_without_body( api_base="https://s3.us-east-1.amazonaws.com/safe-bucket/litellm-bedrock-files-model-id-abc.jsonl", aws_region_name="us-east-1", request_params=request_params, From 4ab5719ff9e0770ecb9f2d1b53c4caf58f19e5db Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 7 Sep 2026 16:36:27 -0700 Subject: [PATCH 065/411] test(batches): use immutable expectations with explicit test doubles --- litellm/files/main.py | 2 +- tests/e2e/batches/test_batch_cleanup.py | 187 ++++++++++++------------ 2 files changed, 93 insertions(+), 96 deletions(-) diff --git a/litellm/files/main.py b/litellm/files/main.py index 19da77b7364..218518eb3cd 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -31,7 +31,7 @@ FileCreateProvider = Literal[ FileRetrieveProvider = Literal[ "openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic" ] -FileDeleteProvider = Literal["openai", "azure", "gemini", "litellm_proxy", "manus", "anthropic"] +FileDeleteProvider = Literal["openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic"] FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic"] import litellm from litellm import get_secret_str diff --git a/tests/e2e/batches/test_batch_cleanup.py b/tests/e2e/batches/test_batch_cleanup.py index a875aee719b..d0038139dcf 100644 --- a/tests/e2e/batches/test_batch_cleanup.py +++ b/tests/e2e/batches/test_batch_cleanup.py @@ -1,7 +1,7 @@ from builtins import ExceptionGroup -from collections.abc import Iterator -from dataclasses import dataclass, field +from collections.abc import Callable from typing import Final +from unittest.mock import Mock, call import pytest from batch_cleanup import BATCH_CANCEL_TIMEOUT_SECONDS, CLEANUP_DELAYS, cleanup_batch, cleanup_file, cleanup_result @@ -15,35 +15,43 @@ MANAGED_FILE_ID: Final = "bGl0ZWxsbV9wcm94eTtmaWxlLTE=" MANAGED_BATCH_ID: Final = "bGl0ZWxsbV9wcm94eTtiYXRjaC0x" -@dataclass(frozen=True, slots=True) class ExpectedCalls[T]: - values: Iterator[T] + def __init__(self, values: tuple[T, ...]) -> None: + self.values: Final = values + self.recorder: Final = Mock() def __call__(self, value: T) -> None: - assert next(self.values, None) == value + self.recorder(value) def assert_done(self) -> None: - assert tuple(self.values) == () + assert tuple(self.recorder.call_args_list) == tuple(call(value) for value in self.values) -@dataclass(frozen=True, slots=True) class CleanupClient: - calls: ExpectedCalls[str] - files: Iterator[Result[FileDeleteResponse]] = field(default_factory=lambda: iter(())) - batches: Iterator[Result[BatchObject]] = field(default_factory=lambda: iter(())) - cancellations: Iterator[Result[BatchObject]] = field(default_factory=lambda: iter(())) + def __init__( + self, + *, + calls: ExpectedCalls[str], + files: tuple[Result[FileDeleteResponse], ...] = (), + batches: tuple[Result[BatchObject], ...] = (), + cancellations: tuple[Result[BatchObject], ...] = (), + ) -> None: + self.calls: Final = calls + self.file_response: Final[Callable[[], Result[FileDeleteResponse]]] = Mock(side_effect=files) + self.batch_response: Final[Callable[[], Result[BatchObject]]] = Mock(side_effect=batches) + self.cancel_response: Final[Callable[[], Result[BatchObject]]] = Mock(side_effect=cancellations) def delete_file(self, file_id: str, *, key: str, provider: str | None = None) -> Result[FileDeleteResponse]: self.calls(f"delete {provider} {file_id}") - return next(self.files) + return self.file_response() def retrieve_batch(self, batch_id: str, *, key: str, provider: str | None = None) -> Result[BatchObject]: self.calls(f"retrieve {provider} {batch_id}") - return next(self.batches) + return self.batch_response() def cancel_batch(self, batch_id: str, *, key: str, provider: str | None = None) -> Result[BatchObject]: self.calls(f"cancel {provider} {batch_id}") - return next(self.cancellations) + return self.cancel_response() def generate_key(self, body: KeyGenerateBody) -> str: return "test-key" @@ -68,17 +76,15 @@ class TestFileCleanup: response: Final = Success( status_code=200, data=FileDeleteResponse.model_validate({"id": MANAGED_FILE_ID, "object": "file"}) ) - client: Final = CleanupClient( - calls=ExpectedCalls(iter((f"delete None {MANAGED_FILE_ID}",))), files=iter((response,)) - ) + client: Final = CleanupClient(calls=ExpectedCalls((f"delete None {MANAGED_FILE_ID}",)), files=(response,)) cleanup_file(client, MANAGED_FILE_ID, key="test-key") client.calls.assert_done() @pytest.mark.parametrize("file_id", ["file-1", MANAGED_FILE_ID]) def test_a_success_status_without_a_deletion_confirmation_is_rejected(self, file_id: str) -> None: client: Final = CleanupClient( - calls=ExpectedCalls(iter((f"delete None {file_id}",))), - files=iter((Success(status_code=200, data=FileDeleteResponse(id=file_id)),)), + calls=ExpectedCalls((f"delete None {file_id}",)), + files=(Success(status_code=200, data=FileDeleteResponse(id=file_id)),), ) with pytest.raises(AssertionError, match="did not confirm deletion"): cleanup_file(client, file_id, key="test-key") @@ -88,15 +94,15 @@ class TestFileCleanup: def test_deletes_raw_files_through_the_upload_provider(self, cap: Capability) -> None: expected_provider: Final = cap.provider if cap.scenario in {"model_param", "provider_fallback"} else None client: Final = CleanupClient( - calls=ExpectedCalls(iter((f"delete {expected_provider} file-1",))), files=iter((deleted_file(),)) + calls=ExpectedCalls((f"delete {expected_provider} file-1",)), files=(deleted_file(),) ) cleanup_file(client, "file-1", key="test-key", provider=cap.file_provider) client.calls.assert_done() def test_failed_delete_is_reported_after_remaining_resources_are_cleaned(self) -> None: client: Final = CleanupClient( - calls=ExpectedCalls(iter(("delete azure file-1", "delete key test-key"))), - files=iter((UnknownApiError(status_code=403, body="secret response"),)), + calls=ExpectedCalls(("delete azure file-1", "delete key test-key")), + files=(UnknownApiError(status_code=403, body="secret response"),), ) manager: Final = ResourceManager(client=client, strict_cleanup=True) key: Final = manager.key() @@ -109,7 +115,7 @@ class TestFileCleanup: def test_success_response_must_confirm_deletion(self) -> None: client: Final = CleanupClient( - calls=ExpectedCalls(iter(("delete None file-1",))), files=iter((deleted_file(deleted=False),)) + calls=ExpectedCalls(("delete None file-1",)), files=(deleted_file(deleted=False),) ) with pytest.raises(AssertionError, match="did not confirm deletion"): cleanup_file(client, "file-1", key="test-key") @@ -117,16 +123,16 @@ class TestFileCleanup: def test_cleanup_is_idempotent_when_file_is_already_deleted(self) -> None: client: Final = CleanupClient( - calls=ExpectedCalls(iter(("delete azure file-1",))), - files=iter((UnknownApiError(status_code=404, body="missing"),)), + calls=ExpectedCalls(("delete azure file-1",)), + files=(UnknownApiError(status_code=404, body="missing"),), ) cleanup_file(client, "file-1", key="test-key", provider="azure") client.calls.assert_done() def test_default_resource_cleanup_keeps_existing_best_effort_behavior(self) -> None: client: Final = CleanupClient( - calls=ExpectedCalls(iter(("delete None file-1", "delete key test-key"))), - files=iter((UnknownApiError(status_code=403, body="forbidden"),)), + calls=ExpectedCalls(("delete None file-1", "delete key test-key")), + files=(UnknownApiError(status_code=403, body="forbidden"),), ) manager: Final = ResourceManager(client=client) key: Final = manager.key() @@ -141,37 +147,39 @@ class TestCleanupRetries: [NetworkError(message="offline"), RateLimitedError(), UnknownApiError(status_code=503, body="unavailable")], ) def test_transient_error_retries_and_returns_success(self, failure: Result[FileDeleteResponse]) -> None: - outcomes: Final = iter((failure, deleted_file())) - delays: Final = ExpectedCalls(iter((1.0,))) - result: Final[Result[FileDeleteResponse]] = cleanup_result(lambda: next(outcomes), wait=delays) + responses: Final = (failure, deleted_file()) + outcomes: Final = Mock(side_effect=responses) + delays: Final = ExpectedCalls((1.0,)) + result: Final[Result[FileDeleteResponse]] = cleanup_result(outcomes, wait=delays) assert isinstance(result, Success) and result.data.deleted delays.assert_done() def test_persistent_error_has_bounded_retries(self) -> None: failure: Final = UnknownApiError(status_code=503, body="unavailable") - outcomes: Final[Iterator[Result[FileDeleteResponse]]] = iter((failure,) * (len(CLEANUP_DELAYS) + 1)) - delays: Final = ExpectedCalls(iter(CLEANUP_DELAYS)) - result: Final[Result[FileDeleteResponse]] = cleanup_result(lambda: next(outcomes), wait=delays) + outcomes: Final = Mock(return_value=failure) + delays: Final = ExpectedCalls(CLEANUP_DELAYS) + result: Final[Result[FileDeleteResponse]] = cleanup_result(outcomes, wait=delays) assert result is failure delays.assert_done() - assert next(outcomes, None) is None + assert outcomes.call_count == len(CLEANUP_DELAYS) + 1 def test_permanent_error_is_not_retried(self) -> None: failure: Final = UnknownApiError(status_code=403, body="forbidden") - outcomes: Final = iter((failure, deleted_file())) - delays: Final = ExpectedCalls[float](iter(())) - assert cleanup_result(lambda: next(outcomes), wait=delays) is failure + responses: Final = (failure, deleted_file()) + outcomes: Final = Mock(side_effect=responses) + delays: Final = ExpectedCalls[float](()) + assert cleanup_result(outcomes, wait=delays) is failure delays.assert_done() - assert isinstance(next(outcomes), Success) + assert outcomes.call_count == 1 class TestBatchCancellation: def test_cancelling_batch_is_polled_until_terminal_without_cancelling_again(self) -> None: client: Final = CleanupClient( - calls=ExpectedCalls(iter((f"retrieve None {MANAGED_BATCH_ID}",) * 3)), - batches=iter((batch("cancelling"), batch("cancelling"), batch("cancelled"))), + calls=ExpectedCalls((f"retrieve None {MANAGED_BATCH_ID}",) * 3), + batches=(batch("cancelling"), batch("cancelling"), batch("cancelled")), ) - delays: Final = ExpectedCalls(iter((10.0,))) + delays: Final = ExpectedCalls((10.0,)) cleanup_batch(client, MANAGED_BATCH_ID, key="test-key", wait=delays) client.calls.assert_done() delays.assert_done() @@ -179,23 +187,22 @@ class TestBatchCancellation: def test_cancellation_timeout_is_reported_but_file_and_key_cleanup_still_run(self) -> None: client: Final = CleanupClient( calls=ExpectedCalls( - iter( - ( - f"retrieve None {MANAGED_BATCH_ID}", - f"retrieve None {MANAGED_BATCH_ID}", - "delete None file-1", - "delete key test-key", - ) + ( + f"retrieve None {MANAGED_BATCH_ID}", + f"retrieve None {MANAGED_BATCH_ID}", + "delete None file-1", + "delete key test-key", ) ), - batches=iter((batch("cancelling"), batch("cancelling"))), - files=iter((deleted_file(),)), + batches=(batch("cancelling"), batch("cancelling")), + files=(deleted_file(),), ) - ticks: Final = iter((0.0, BATCH_CANCEL_TIMEOUT_SECONDS)) + times: Final = (0.0, BATCH_CANCEL_TIMEOUT_SECONDS) + ticks: Final[Callable[[], float]] = Mock(side_effect=times) manager: Final = ResourceManager(client=client, strict_cleanup=True) key: Final = manager.key() manager.defer(lambda: cleanup_file(client, "file-1", key=key)) - manager.defer(lambda: cleanup_batch(client, MANAGED_BATCH_ID, key=key, clock=lambda: next(ticks))) + manager.defer(lambda: cleanup_batch(client, MANAGED_BATCH_ID, key=key, clock=ticks)) with pytest.raises(ExceptionGroup) as caught: manager.teardown() assert "cancellation did not finish" in str(caught.value.exceptions[0]) @@ -203,17 +210,15 @@ class TestBatchCancellation: @pytest.mark.parametrize("status", ["completed", "failed", "expired", "cancelled"]) def test_inactive_batch_needs_no_cancellation(self, status: str) -> None: - client: Final = CleanupClient( - calls=ExpectedCalls(iter(("retrieve None batch-1",))), batches=iter((batch(status),)) - ) + client: Final = CleanupClient(calls=ExpectedCalls(("retrieve None batch-1",)), batches=(batch(status),)) cleanup_batch(client, "batch-1", key="test-key") client.calls.assert_done() def test_active_batch_is_cancelled_through_its_provider(self) -> None: client: Final = CleanupClient( - calls=ExpectedCalls(iter(("retrieve azure batch-1", "cancel azure batch-1"))), - batches=iter((batch("in_progress"), batch("cancelled"))), - cancellations=iter((batch("cancelling"),)), + calls=ExpectedCalls(("retrieve azure batch-1", "cancel azure batch-1")), + batches=(batch("in_progress"), batch("cancelled")), + cancellations=(batch("cancelling"),), ) cleanup_batch(client, "batch-1", key="test-key", provider="azure") client.calls.assert_done() @@ -225,23 +230,21 @@ class TestBatchCancellation: ) -> None: client: Final = CleanupClient( calls=ExpectedCalls( - iter( - ( - f"retrieve vertex_ai {batch_id}", - f"cancel vertex_ai {batch_id}", - f"retrieve vertex_ai {batch_id}", - f"retrieve vertex_ai {batch_id}", - f"retrieve vertex_ai {batch_id}", - "delete vertex_ai file-1", - "delete key test-key", - ) + ( + f"retrieve vertex_ai {batch_id}", + f"cancel vertex_ai {batch_id}", + f"retrieve vertex_ai {batch_id}", + f"retrieve vertex_ai {batch_id}", + f"retrieve vertex_ai {batch_id}", + "delete vertex_ai file-1", + "delete key test-key", ) ), - batches=iter((batch("validating"), batch(pending_status), batch(pending_status), batch("cancelled"))), - cancellations=iter((batch(pending_status),)), - files=iter((deleted_file(),)), + batches=(batch("validating"), batch(pending_status), batch(pending_status), batch("cancelled")), + cancellations=(batch(pending_status),), + files=(deleted_file(),), ) - delays: Final = ExpectedCalls(iter((10.0, 10.0))) + delays: Final = ExpectedCalls((10.0, 10.0)) manager: Final = ResourceManager(client=client, strict_cleanup=True) key: Final = manager.key() manager.defer(lambda: cleanup_file(client, "file-1", key=key, provider="vertex_ai")) @@ -255,28 +258,22 @@ class TestBatchCancellation: self, output_delete_fails: bool ) -> None: client: Final = CleanupClient( - calls=ExpectedCalls( - iter(("retrieve openai batch-1", "delete openai file-output", "delete openai file-error")) - ), - batches=iter( - ( - Success( - status_code=200, - data=BatchObject( - id="batch-1", - status="completed", - input_file_id="file-input", - output_file_id="file-output", - error_file_id="file-error", - ), + calls=ExpectedCalls(("retrieve openai batch-1", "delete openai file-output", "delete openai file-error")), + batches=( + Success( + status_code=200, + data=BatchObject( + id="batch-1", + status="completed", + input_file_id="file-input", + output_file_id="file-output", + error_file_id="file-error", ), - ) + ), ), - files=iter( - ( - UnknownApiError(status_code=403, body="forbidden") if output_delete_fails else deleted_file(), - deleted_file(), - ) + files=( + UnknownApiError(status_code=403, body="forbidden") if output_delete_fails else deleted_file(), + deleted_file(), ), ) if output_delete_fails: @@ -289,9 +286,9 @@ class TestBatchCancellation: @pytest.mark.parametrize("status", ["completed", "in_progress"]) def test_cancellation_conflict_is_accepted_only_when_batch_became_inactive(self, status: str) -> None: client: Final = CleanupClient( - calls=ExpectedCalls(iter(("retrieve None batch-1", "cancel None batch-1", "retrieve None batch-1"))), - batches=iter((batch("in_progress"), batch(status))), - cancellations=iter((UnknownApiError(status_code=409, body="conflict"),)), + calls=ExpectedCalls(("retrieve None batch-1", "cancel None batch-1", "retrieve None batch-1")), + batches=(batch("in_progress"), batch(status)), + cancellations=(UnknownApiError(status_code=409, body="conflict"),), ) if status == "completed": cleanup_batch(client, "batch-1", key="test-key") From 5c23d296d31c3eba4a362fd0fa0e8cca96f51426 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:51:56 -0700 Subject: [PATCH 066/411] fix(azure_ai): cost OCR relays per page so non-chat relays debit budgets --- .../azure_ai/passthrough/transformation.py | 45 +++++- .../base_llm/passthrough/transformation.py | 3 +- ...est_azure_ai_passthrough_transformation.py | 134 +++++++++++++++++- 3 files changed, 172 insertions(+), 10 deletions(-) diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index 4776bbe0755..4cb93b4f8f9 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -1,25 +1,28 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from litellm._logging import verbose_logger from litellm.llms.azure_ai.common_utils import ( AzureFoundryModelInfo, api_key_header_for_base, get_azure_ai_auth_headers, ) +from litellm.llms.azure_ai.ocr.common_utils import get_azure_ai_ocr_config from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, strip_leading_model_segment from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import StandardPassThroughResponseObject +from litellm.types.utils import CallTypes, StandardPassThroughResponseObject if TYPE_CHECKING: from httpx import URL, Response from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.types.utils import CostResponseTypes @@ -75,6 +78,10 @@ def relayed_body(httpx_response: Response) -> str | dict: class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): + def __init__(self, ocr_config_for: Callable[[str], BaseOCRConfig | None] = get_azure_ai_ocr_config) -> None: + super().__init__() + self.ocr_config_for: Final = ocr_config_for + def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool: return bool(request_data.get("stream")) @@ -123,7 +130,7 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): request_data: Mapping[str, object], logging_obj: Logging, endpoint: str, - ) -> CostResponseTypes | StandardPassThroughResponseObject | None: + ) -> CostResponseTypes | OCRResponse | StandardPassThroughResponseObject | None: from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig chat_result: Final = AzurePassthroughConfig().logging_non_streaming_response( # pyright: ignore[reportUnknownMemberType] # the Azure config still types request_data as a bare dict @@ -136,8 +143,40 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): ) if chat_result is not None: return chat_result + ocr_result: Final = self.logged_ocr_response(model, httpx_response, logging_obj, endpoint) + if ocr_result is not None: + return ocr_result return StandardPassThroughResponseObject(response=relayed_body(httpx_response)) + def logged_ocr_response( + self, model: str, httpx_response: Response, logging_obj: Logging, endpoint: str + ) -> OCRResponse | None: + ocr_config: Final = self.ocr_config_for(model) + if ocr_config is None or httpx_response.status_code != 200: + return None + relayed_url: Final = httpx_response.request.url + relayed_origin: Final = str(relayed_url.copy_with(path="/", query=None, fragment=None)).rstrip("/") + ocr_url: Final = httpx.URL( + ocr_config.get_complete_url( + api_base=relayed_origin, + model=model, + optional_params={}, # mutable-ok: BaseOCRConfig wants a dict + ) + ) + known_prefixes: Final = (model, model_group_from(logging_obj.litellm_params)) + native_endpoint: Final = strip_leading_model_segment(endpoint, known_prefixes) + if f"/{native_endpoint.strip('/')}" != ocr_url.path: + return None + try: + ocr_response: Final = ocr_config.transform_ocr_response( + model=model, raw_response=httpx_response, logging_obj=logging_obj + ) + except (ValueError, AttributeError) as error: + verbose_logger.warning("azure_ai passthrough: OCR body from %s is not costable: %s", ocr_url, error) + return None + logging_obj.call_type = CallTypes.aocr.value # rebind-ok: routes cost calculation to the per-page OCR path + return ocr_response + def handle_logging_collected_chunks( self, all_chunks: Sequence[str], diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index d97ecddc75d..464c121ecc1 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -11,6 +11,7 @@ if TYPE_CHECKING: from litellm.types.utils import CostResponseTypes, StandardPassThroughResponseObject from ..chat.transformation import BaseLLMException + from ..ocr.transformation import OCRResponse def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> str: @@ -120,7 +121,7 @@ class BasePassthroughConfig(BaseLLMModelInfo): request_data: dict, logging_obj: "LiteLLMLoggingObj", endpoint: str, - ) -> Optional["CostResponseTypes | StandardPassThroughResponseObject"]: + ) -> Optional["CostResponseTypes | OCRResponse | StandardPassThroughResponseObject"]: pass def handle_logging_collected_chunks( diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index e343d05c5df..644316c2e37 100644 --- a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -1,11 +1,14 @@ import json +from datetime import datetime from unittest.mock import MagicMock import httpx import pytest import litellm +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.passthrough.transformation import AzureAIPassthroughConfig +from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.types.utils import LlmProviders, ModelResponse from litellm.utils import ProviderConfigManager @@ -238,16 +241,135 @@ def _non_chat_logging_result(content: bytes, content_type: str): ) -def test_non_chat_relay_logs_the_parsed_body_so_spend_tracking_sees_the_call(): - result = _non_chat_logging_result(b'{"id":"parse-1","pages":[],"meta":{"billed_units":{"pages":1}}}', "application/json") - - assert result == {"response": {"id": "parse-1", "pages": [], "meta": {"billed_units": {"pages": 1}}}} - - def test_non_chat_relay_with_a_non_json_body_logs_the_raw_text(): assert _non_chat_logging_result(b"page one", "text/plain") == {"response": "page one"} +def _relay_logging_obj(model: str, api_base: str) -> Logging: + logging_obj = Logging( + model=model, + messages=[], + stream=False, + call_type="allm_passthrough_route", + start_time=datetime.now(), + litellm_call_id="call-1", + function_id="fn-1", + ) + logging_obj.update_environment_variables( + model=model, + litellm_params={"api_base": api_base, "custom_llm_provider": "azure_ai"}, + optional_params={}, + custom_llm_provider="azure_ai", + ) + return logging_obj + + +def _relay_logging_result( + config: AzureAIPassthroughConfig, model: str, native_path: str, body, api_base: str = FOUNDRY_BASE, status_code: int = 200 +): + relayed_url = f"{FOUNDRY_BASE}/{native_path}?api-version=2024-05-01-preview" + logging_obj = _relay_logging_obj(model, api_base) + response = httpx.Response( + status_code=status_code, + headers={"content-type": "application/json"}, + content=json.dumps(body).encode("utf-8"), + request=httpx.Request("POST", relayed_url), + ) + result = config.logging_non_streaming_response( + model=model, + custom_llm_provider="azure_ai", + httpx_response=response, + request_data={"model": model}, + logging_obj=logging_obj, + endpoint=f"{model}/{native_path}", + ) + return result, logging_obj + + +MISTRAL_OCR_BODY = { + "pages": [{"index": 0, "markdown": "page one"}, {"index": 1, "markdown": "page two"}], + "model": "mistral-document-ai-2512", + "usage_info": {"pages_processed": 2, "doc_size_bytes": 4321}, +} + + +def test_mistral_document_ai_relay_is_costed_per_page(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), "mistral-document-ai-2512", "providers/mistral/azure/ocr", MISTRAL_OCR_BODY + ) + per_page = litellm.get_model_info("azure_ai/mistral-document-ai-2512")["ocr_cost_per_page"] + + assert isinstance(result, OCRResponse) + assert result.usage_info.pages_processed == 2 + assert per_page > 0 + assert logging_obj._response_cost_calculator(result=result) == pytest.approx(2 * per_page) + + +def test_ocr_route_under_a_models_api_base_is_still_recognised(): + result, _ = _relay_logging_result( + AzureAIPassthroughConfig(), + "mistral-document-ai-2512", + "providers/mistral/azure/ocr", + MISTRAL_OCR_BODY, + api_base=f"{FOUNDRY_BASE}/models", + ) + + assert isinstance(result, OCRResponse) + + +def test_relay_to_a_non_ocr_route_keeps_the_passthrough_object_and_call_type(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), "mistral-document-ai-2512", "models/info", {"name": "mistral-document-ai-2512"} + ) + + assert result == {"response": {"name": "mistral-document-ai-2512"}} + assert logging_obj.call_type == "allm_passthrough_route" + + +COHERE_PARSE_BODY = {"id": "parse-1", "pages": [], "meta": {"billed_units": {"pages": 3}}} + + +def test_cohere_parse_relay_is_costed_per_billed_page(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), "Cohere-parse-v5", "providers/cohere/v2/parse", COHERE_PARSE_BODY + ) + per_page = litellm.get_model_info("azure_ai/Cohere-parse-v5")["ocr_cost_per_page"] + + assert isinstance(result, OCRResponse) + assert result.usage_info.pages_processed == 3 + assert logging_obj.call_type == "aocr" + assert per_page > 0 + assert logging_obj._response_cost_calculator(result=result) == pytest.approx(3 * per_page) + + +def test_deployment_without_an_ocr_config_is_never_costed_as_ocr(): + config = AzureAIPassthroughConfig(ocr_config_for=lambda model: None) + result, logging_obj = _relay_logging_result( + config, "mistral-document-ai-2512", "providers/mistral/azure/ocr", MISTRAL_OCR_BODY + ) + + assert result == {"response": MISTRAL_OCR_BODY} + assert logging_obj.call_type == "allm_passthrough_route" + + +def test_accepted_ocr_job_without_a_result_body_is_not_costed(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), "mistral-document-ai-2512", "providers/mistral/azure/ocr", {"status": "running"}, status_code=202 + ) + + assert result == {"response": {"status": "running"}} + assert logging_obj.call_type == "allm_passthrough_route" + + +def test_unparseable_ocr_body_falls_back_to_the_passthrough_object(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), "mistral-document-ai-2512", "providers/mistral/azure/ocr", ["not", "an", "ocr", "body"] + ) + + assert result == {"response": '["not", "an", "ocr", "body"]'} + assert logging_obj.call_type == "allm_passthrough_route" + + def test_streaming_chat_completion_chunks_are_costed_like_azure(): head = {"id": "chatcmpl-1", "object": "chat.completion.chunk", "created": 1, "model": "gpt-5.4-mini"} chunks = [ From 43a02e2dbcd485c3cc5dd31d8b11161da650c356 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 7 Sep 2026 16:53:41 -0700 Subject: [PATCH 067/411] fix(proxy): report the billed token rates in /cost/estimate The rate fields reported base cost-map prices while the cost lines were billed at the token tier, off-peak window and regional multipliers the calculator picks for the request, so a line did not always equal tokens times its reported rate. get_billed_token_rates now resolves the rates once, the token-type breakdown and the endpoint both read from it, and a tiered-model test asserts every line equals its token count times the rate reported next to it Claude-Session: https://claude.ai/code/session_011Tn3657NkV6ojLqewL64Kb --- .../litellm_core_utils/llm_cost_calc/utils.py | 206 ++++++++++++------ litellm/proxy/_types.py | 6 +- .../cost_tracking_settings.py | 63 ++---- .../llm_cost_calc/test_llm_cost_calc_utils.py | 49 +++++ .../test_cost_tracking_settings.py | 52 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 +- 6 files changed, 258 insertions(+), 128 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 8d53c8da3e6..2bb15fb4c48 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1329,16 +1329,118 @@ def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetai ) -def _custom_pricing_token_type_breakdown(usage: Usage, custom_cost_per_token: CostPerToken) -> TokenTypeCostBreakdown: +@dataclass(frozen=True, slots=True) +class BilledTokenRates: + """Per-token rates one request's usage bills at, after token tiers, off-peak windows and the + regional multipliers the totals apply, so each cost line equals its token count times its rate.""" + + input_cost_per_token: float + output_cost_per_token: float + cache_read_input_token_cost: float + cache_creation_input_token_cost: float + cache_creation_input_token_cost_above_1hr: float + output_cost_per_reasoning_token: float + + def scaled(self, multiplier: float) -> "BilledTokenRates": + if multiplier == 1.0: + return self + return BilledTokenRates( + input_cost_per_token=self.input_cost_per_token * multiplier, + output_cost_per_token=self.output_cost_per_token * multiplier, + cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier, + cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier, + cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier, + output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, + ) + + +def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRates: """Flat custom pricing has no tiers, uplifts or reasoning rate: cache tokens bill at the configured cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does.""" input_rate: Final = custom_cost_per_token["input_cost_per_token"] - cache_read_tokens, cache_creation_tokens, _ = _cache_token_counts(usage) - return TokenTypeCostBreakdown( - reasoning_cost=float(_reasoning_token_count(usage)) * custom_cost_per_token["output_cost_per_token"], - cache_read_cost=float(cache_read_tokens) * custom_cost_per_token.get("cache_read_input_token_cost", input_rate), - cache_creation_cost=float(cache_creation_tokens) - * custom_cost_per_token.get("cache_creation_input_token_cost", input_rate), + output_rate: Final = custom_cost_per_token["output_cost_per_token"] + cache_creation_rate: Final = custom_cost_per_token.get("cache_creation_input_token_cost", input_rate) + return BilledTokenRates( + input_cost_per_token=input_rate, + output_cost_per_token=output_rate, + cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate), + cache_creation_input_token_cost=cache_creation_rate, + cache_creation_input_token_cost_above_1hr=cache_creation_rate, + output_cost_per_reasoning_token=output_rate, + ) + + +def _cost_map_billed_rates( + model_info: ModelInfo, + usage: Usage, + custom_llm_provider: str | None, + service_tier: str | None, + data_residency: str | None, + vertex_location: str | None, + current_time: datetime | None, +) -> BilledTokenRates: + billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc) + ( + prompt_base_cost, + completion_base_cost, + cache_creation_cost_rate, + cache_creation_cost_above_1hr_rate, + cache_read_cost_rate, + ) = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier=service_tier, + current_time=billing_time, + threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), + ) + reasoning_rate: Final = _resolve_billed_reasoning_rate( + model_info=model_info, + usage=usage, + service_tier=service_tier, + completion_base_cost=completion_base_cost, + current_time=billing_time, + ) + multiplier: Final = ( + _get_regional_uplift_multiplier(model_info, data_residency) + * get_vertex_regional_endpoint_uplift(model_info, vertex_location) + * get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) + ) + return BilledTokenRates( + input_cost_per_token=prompt_base_cost, + output_cost_per_token=completion_base_cost, + cache_read_input_token_cost=cache_read_cost_rate, + cache_creation_input_token_cost=cache_creation_cost_rate, + cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate, + output_cost_per_reasoning_token=reasoning_rate, + ).scaled(multiplier) + + +def get_billed_token_rates( + model: str, + custom_llm_provider: str | None, + usage: Usage, + service_tier: str | None = None, + data_residency: str | None = None, + vertex_location: str | None = None, + current_time: datetime | None = None, + custom_cost_per_token: CostPerToken | None = None, +) -> BilledTokenRates | None: + """Rates the cost calculator bills ``usage`` at, resolved exactly as the totals and the token-type + breakdown resolve them. None when the model's pricing cannot be resolved.""" + if custom_cost_per_token is not None: + return _custom_pricing_rates(custom_cost_per_token) + try: + model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: + return None + return _cost_map_billed_rates( + model_info=model_info, + usage=usage, + custom_llm_provider=custom_llm_provider, + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + current_time=current_time, ) @@ -1360,77 +1462,39 @@ def get_token_type_cost_breakdown( cost calculators bypass ``generic_cost_per_token``, because cache tokens always land on ``prompt_tokens_details`` (via the Usage constructor and provider transformations) and reasoning tokens on ``completion_tokens_details``. It reuses - the same rate-resolution primitives as the total-cost path so the breakdown can - never drift from the totals. A deployment billed by ``custom_cost_per_token`` is - priced from those flat rates instead of the cost map, for the same reason. + the same rate resolution as the total-cost path (``get_billed_token_rates``) so the + breakdown can never drift from the totals. A deployment billed by + ``custom_cost_per_token`` is priced from those flat rates instead of the cost map and, + like its totals, bills cache writes flat rather than by their 5m/1h split. Returns zeros (never raises) when the model or its pricing cannot be resolved. """ - if custom_cost_per_token is not None: - return _custom_pricing_token_type_breakdown(usage=usage, custom_cost_per_token=custom_cost_per_token) - - try: - model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) - except Exception: + rates: Final = get_billed_token_rates( + model=model, + custom_llm_provider=custom_llm_provider, + usage=usage, + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + current_time=current_time, + custom_cost_per_token=custom_cost_per_token, + ) + if rates is None: return TokenTypeCostBreakdown(0.0, 0.0, 0.0) - billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc) - ( - _prompt_base_cost, - completion_base_cost, - cache_creation_cost_rate, - cache_creation_cost_above_1hr_rate, - cache_read_cost_rate, - ) = _get_token_base_cost( - model_info=model_info, - usage=usage, - service_tier=service_tier, - current_time=billing_time, - threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), - ) - - reasoning_rate: Final = _resolve_billed_reasoning_rate( - model_info=model_info, - usage=usage, - service_tier=service_tier, - completion_base_cost=completion_base_cost, - current_time=billing_time, - ) - reasoning_cost = float(_reasoning_token_count(usage)) * reasoning_rate - cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage) - cache_read_cost = float(cache_read_tokens) * cache_read_cost_rate - cache_creation_cost = calculate_cache_writing_cost( - cache_creation_tokens=cache_creation_tokens, - cache_creation_token_details=cache_creation_token_details, - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr_rate, - cache_creation_cost=cache_creation_cost_rate, + cache_creation_cost: Final = ( + float(cache_creation_tokens) * rates.cache_creation_input_token_cost + if custom_cost_per_token is not None + else calculate_cache_writing_cost( + cache_creation_tokens=cache_creation_tokens, + cache_creation_token_details=cache_creation_token_details, + cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr, + cache_creation_cost=rates.cache_creation_input_token_cost, + ) ) - - # Apply the same flat regional-processing uplift the totals get, so per-type - # costs stay reconciled with input_cost/output_cost for regionalized OpenAI hosts. - uplift: Final = _get_regional_uplift_multiplier(model_info, data_residency) - if uplift != 1.0: - reasoning_cost *= uplift - cache_read_cost *= uplift - cache_creation_cost *= uplift - - vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) - if vertex_uplift != 1.0: - reasoning_cost *= vertex_uplift - cache_read_cost *= vertex_uplift - cache_creation_cost *= vertex_uplift - - # Mirror the provider-specific geo uplift (e.g. Anthropic us: 1.1) the totals - # apply, so cache and reasoning line items stay reconciled with them. - geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) - if geo_multiplier != 1.0: - reasoning_cost *= geo_multiplier - cache_read_cost *= geo_multiplier - cache_creation_cost *= geo_multiplier - return TokenTypeCostBreakdown( - reasoning_cost=reasoning_cost, - cache_read_cost=cache_read_cost, + reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, + cache_read_cost=float(cache_read_tokens) * rates.cache_read_input_token_cost, cache_creation_cost=cache_creation_cost, ) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 65b7d409d7c..3c1cd9bfc69 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5197,9 +5197,9 @@ class CostEstimateResponse(LiteLLMPydanticObjectBase): default=None, description="Cache-write share of monthly_input_cost" ) monthly_reasoning_cost: float | None = Field(default=None, description="Reasoning share of monthly_output_cost") - # Pricing info - input_cost_per_token: float | None = None - output_cost_per_token: float | None = None + # Pricing info: the rates this request's usage bills at, after token tiers and regional multipliers + input_cost_per_token: float | None = Field(default=None, description="Rate billed per input token") + output_cost_per_token: float | None = Field(default=None, description="Rate billed per output token") cache_read_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-read token") cache_creation_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-write token") output_cost_per_reasoning_token: float | None = Field(default=None, description="Rate billed per reasoning token") diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 51f31a757cd..17d82fd17e3 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -20,6 +20,7 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger from litellm.cost_calculator import completion_cost +from litellm.litellm_core_utils.llm_cost_calc.utils import get_billed_token_rates from litellm.proxy._types import ( CommonProxyErrors, CostEstimateRequest, @@ -170,44 +171,6 @@ def _cost_lines(cost_per_request: float, cost_breakdown: CostBreakdown | None) - ) -@dataclass(frozen=True, slots=True) -class EffectiveTokenRates: - input_cost_per_token: float | None - output_cost_per_token: float | None - cache_read_input_token_cost: float | None - cache_creation_input_token_cost: float | None - output_cost_per_reasoning_token: float | None - - -def _custom_token_rates(custom_cost_per_token: CostPerToken) -> EffectiveTokenRates: - input_rate: Final = custom_cost_per_token["input_cost_per_token"] - output_rate: Final = custom_cost_per_token["output_cost_per_token"] - return EffectiveTokenRates( - input_cost_per_token=input_rate, - output_cost_per_token=output_rate, - cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate), - cache_creation_input_token_cost=custom_cost_per_token.get("cache_creation_input_token_cost", input_rate), - output_cost_per_reasoning_token=output_rate, - ) - - -def _cost_map_token_rates(model_info: ModelInfo | None) -> EffectiveTokenRates: - """Base rates the cost calculator bills flat usage at: a cost-map model without a cache price - bills cache tokens at zero, and one without a reasoning price bills reasoning at the output rate.""" - if model_info is None: - return EffectiveTokenRates(None, None, None, None, None) - sources: Final = (model_info,) - output_rate: Final = _configured_price("output_cost_per_token", sources) - reasoning_rate: Final = _configured_price("output_cost_per_reasoning_token", sources) - return EffectiveTokenRates( - input_cost_per_token=_configured_price("input_cost_per_token", sources), - output_cost_per_token=output_rate, - cache_read_input_token_cost=_configured_price("cache_read_input_token_cost", sources) or 0.0, - cache_creation_input_token_cost=_configured_price("cache_creation_input_token_cost", sources) or 0.0, - output_cost_per_reasoning_token=output_rate if reasoning_rate is None else reasoning_rate, - ) - - def _usage_for_estimate(request: CostEstimateRequest) -> Usage: cache_tokens: Final = request.cache_read_input_tokens + request.cache_creation_input_tokens return Usage( @@ -654,7 +617,8 @@ async def estimate_cost( verbose_proxy_logger.debug("Cost estimate: request.model='%s' resolved to '%s'", request.model, resolved_model) - mock_response: Final = ModelResponse(model=resolved_model, usage=_usage_for_estimate(request)) + usage: Final = _usage_for_estimate(request) + mock_response: Final = ModelResponse(model=resolved_model, usage=usage) # Create a logging object to capture cost breakdown litellm_logging_obj: Final = LiteLLMLoggingObj( @@ -688,12 +652,13 @@ async def estimate_cost( daily: Final = per_request.times(request.num_requests_per_day) monthly: Final = per_request.times(request.num_requests_per_month) - model_info: Final = _lookup_model_info(resolved_model) - rates: Final = ( - _custom_token_rates(resolved.custom_cost_per_token) - if resolved.custom_cost_per_token is not None - else _cost_map_token_rates(model_info) + rates: Final = get_billed_token_rates( + model=resolved_model, + custom_llm_provider=resolved_provider, + usage=usage, + custom_cost_per_token=resolved.custom_cost_per_token, ) + model_info: Final = _lookup_model_info(resolved_model) mapped_provider: Final = model_info.get("litellm_provider") if model_info is not None else None custom_llm_provider: Final = mapped_provider if mapped_provider is not None else resolved_provider @@ -727,10 +692,10 @@ async def estimate_cost( monthly_cache_read_cost=monthly.cache_read_cost if monthly is not None else None, monthly_cache_creation_cost=monthly.cache_creation_cost if monthly is not None else None, monthly_reasoning_cost=monthly.reasoning_cost if monthly is not None else None, - input_cost_per_token=rates.input_cost_per_token, - output_cost_per_token=rates.output_cost_per_token, - cache_read_input_token_cost=rates.cache_read_input_token_cost, - cache_creation_input_token_cost=rates.cache_creation_input_token_cost, - output_cost_per_reasoning_token=rates.output_cost_per_reasoning_token, + input_cost_per_token=rates.input_cost_per_token if rates is not None else None, + output_cost_per_token=rates.output_cost_per_token if rates is not None else None, + cache_read_input_token_cost=rates.cache_read_input_token_cost if rates is not None else None, + cache_creation_input_token_cost=rates.cache_creation_input_token_cost if rates is not None else None, + output_cost_per_reasoning_token=rates.output_cost_per_reasoning_token if rates is not None else None, provider=custom_llm_provider, ) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 7065003839b..4de3c059e63 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -27,6 +27,7 @@ from litellm.types.utils import ( ) from litellm.litellm_core_utils.llm_cost_calc.utils import ( + BilledTokenRates, CostCalculatorUtils, PromptTokensDetailsResult, TokenRates, @@ -38,6 +39,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( apply_off_peak_pricing, calculate_cache_writing_cost, generic_cost_per_token, + get_billed_token_rates, get_token_type_cost_breakdown, ) from litellm.types.utils import CacheCreationTokenDetails, Usage @@ -3938,6 +3940,53 @@ def test_token_type_cost_breakdown_reconciles_with_custom_pricing_totals(): assert 300 * 2e-6 + breakdown.reasoning_cost == pytest.approx(completion_cost) +def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "tiered-cache-model", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "cache_creation_input_token_cost": 3.75e-6, + "input_cost_per_token_above_200k_tokens": 6e-6, + "output_cost_per_token_above_200k_tokens": 3e-5, + "cache_read_input_token_cost_above_200k_tokens": 6e-7, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-6, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200_000, cache_creation_tokens=10_000), + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200), + ) + + rates = get_billed_token_rates(model="tiered-cache-model", custom_llm_provider="openai", usage=usage) + breakdown = get_token_type_cost_breakdown(model="tiered-cache-model", custom_llm_provider="openai", usage=usage) + + assert rates == BilledTokenRates( + input_cost_per_token=6e-6, + output_cost_per_token=3e-5, + cache_read_input_token_cost=6e-7, + cache_creation_input_token_cost=7.5e-6, + cache_creation_input_token_cost_above_1hr=0.0, + output_cost_per_reasoning_token=3e-5, + ) + assert breakdown.cache_read_cost == pytest.approx(200_000 * rates.cache_read_input_token_cost) + assert breakdown.cache_creation_cost == pytest.approx(10_000 * rates.cache_creation_input_token_cost) + assert breakdown.reasoning_cost == pytest.approx(200 * rates.output_cost_per_reasoning_token) + + +def test_billed_token_rates_are_none_for_an_unpriced_model(): + usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + + assert get_billed_token_rates(model="no-such-model-anywhere", custom_llm_provider="openai", usage=usage) is None + + def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost_map): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index 6d3ef2ffea9..9847c4092df 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -813,9 +813,7 @@ async def _estimate(mock_router: MagicMock | None, model: str = AN_ALIAS, **over request = CostEstimateRequest( model=model, - input_tokens=INPUT_TOKENS, - output_tokens=OUTPUT_TOKENS, - **overrides, + **{"input_tokens": INPUT_TOKENS, "output_tokens": OUTPUT_TOKENS, **overrides}, ) with patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point "litellm.proxy.proxy_server.llm_router", mock_router @@ -1062,6 +1060,54 @@ class TestEstimateCostCacheAndReasoningTokens: assert response.cache_read_input_token_cost == pytest.approx(5e-7) assert response.cache_creation_input_token_cost == pytest.approx(6.25e-6) + @pytest.mark.asyncio + async def test_a_tiered_model_reports_the_rates_its_lines_were_billed_at(self, monkeypatch): + """Above a token tier the calculator bills every line at the tier's rate, so the reported + rates must be the tier's too: each line equals its token count times the rate next to it.""" + monkeypatch.setitem( + litellm.model_cost, + A_MAPPED_MODEL, + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "cache_creation_input_token_cost": 3.75e-6, + "input_cost_per_token_above_200k_tokens": 6e-6, + "output_cost_per_token_above_200k_tokens": 3e-5, + "cache_read_input_token_cost_above_200k_tokens": 6e-7, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-6, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + + response = await _estimate( + None, + model=A_MAPPED_MODEL, + input_tokens=250_000, + cache_read_input_tokens=200_000, + cache_creation_input_tokens=10_000, + output_tokens=1_000, + reasoning_tokens=200, + ) + + assert response.input_cost_per_token == pytest.approx(6e-6) + assert response.output_cost_per_token == pytest.approx(3e-5) + assert response.cache_read_input_token_cost == pytest.approx(6e-7) + assert response.cache_creation_input_token_cost == pytest.approx(7.5e-6) + assert response.output_cost_per_reasoning_token == pytest.approx(3e-5) + assert response.cache_read_cost_per_request == pytest.approx(200_000 * response.cache_read_input_token_cost) + assert response.cache_creation_cost_per_request == pytest.approx( + 10_000 * response.cache_creation_input_token_cost + ) + assert response.reasoning_cost_per_request == pytest.approx(200 * response.output_cost_per_reasoning_token) + assert response.input_cost_per_request == pytest.approx( + 40_000 * response.input_cost_per_token + + response.cache_read_cost_per_request + + response.cache_creation_cost_per_request + ) + assert response.output_cost_per_request == pytest.approx(1_000 * response.output_cost_per_token) + class TestCostEstimateRequestTokenSubsets: def test_cache_tokens_beyond_the_input_tokens_are_rejected(self): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6a9c86cc1b5..46cb5ad892a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26523,7 +26523,10 @@ export interface components { * @description Input token cost per request (before margin) */ input_cost_per_request: number; - /** Input Cost Per Token */ + /** + * Input Cost Per Token + * @description Rate billed per input token + */ input_cost_per_token?: number | null; /** Input Tokens */ input_tokens: number; @@ -26584,7 +26587,10 @@ export interface components { * @description Output token cost per request (before margin) */ output_cost_per_request: number; - /** Output Cost Per Token */ + /** + * Output Cost Per Token + * @description Rate billed per output token + */ output_cost_per_token?: number | null; /** Output Tokens */ output_tokens: number; From 7d3b68fea5a6343781401e52f40b02f6c581541c Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 7 Sep 2026 16:59:53 -0700 Subject: [PATCH 068/411] fix(files): preserve managed deletion routing and response identity --- .../proxy/hooks/managed_files.py | 13 ++- tests/e2e/batches/COVERAGE.md | 3 + .../proxy/test_managed_files_hook.py | 104 ++++++++++++++++++ 3 files changed, 118 insertions(+), 2 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index bc1eb6cebc2..6e0bb0da3f4 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -1779,7 +1779,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Remove conflicting keys from data to avoid duplicate keyword arguments filtered_data = {k: v for k, v in data.items() if k not in ("model", "file_id")} for model_id, model_file_id in specific_model_file_id_mapping.items(): - delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **filtered_data) # type: ignore + credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id) + delete_data = { + **{k: v for k, v in filtered_data.items() if k != "_litellm_internal_model_credentials"}, + **( + {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))} + if credentials is not None + else {} + ), + } + delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **delete_data) stored_file_object = await self.delete_unified_file_id(file_id, litellm_parent_otel_span) @@ -1790,7 +1799,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): prom_logger.record_managed_file_deleted(result="success") if stored_file_object: - return stored_file_object + return OpenAIFileObject.model_validate(stored_file_object).model_copy(update={"id": file_id}) elif delete_response: delete_response.id = file_id return delete_response diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index ca44fc95e25..919c39f21a2 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -138,6 +138,9 @@ output and error files returned by terminal batches. Bedrock deletion uses a sig restricted to the configured storage buckets and managed file prefixes. The low-RPM test submits with its restricted key and cleans up with the test administrator key +Managed deletion forwards the deployment's trusted bucket configuration and returns +the requested managed file ID even when stored output metadata carries a provider ID + Azure input uploads request `expires_after` anchored to `created_at` with `seconds=1209600`, and the lifecycle tests check the returned expiry. This is a fallback for interrupted runs: immediate deletion remains the normal cleanup. diff --git a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py index 091b958d7c3..48fceb50403 100644 --- a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py +++ b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py @@ -1095,6 +1095,110 @@ async def test_afile_content_passes_trusted_model_credentials_to_router(): assert trusted_credentials["s3_bucket_name"] == "my-bucket" +def _managed_deletion_file_id(provider_file_id): + from litellm.types.utils import SpecialEnums + + value = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( + "application/json", "test-file", "batch-model", provider_file_id, "model-123" + ) + return base64.urlsafe_b64encode(value.encode()).decode().rstrip("=") + + +def _managed_files_with_deletion_row(unified_file_id, provider_file_id, file_object): + from litellm.caching import DualCache + from litellm.models.managed_files import LiteLLM_ManagedFileTable + from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles + + row = LiteLLM_ManagedFileTable( + unified_file_id=unified_file_id, + model_mappings={"model-123": provider_file_id}, + flat_model_file_ids=[provider_file_id], + file_object=file_object, + ) + table = MagicMock( + find_first=AsyncMock(return_value=row), + delete=AsyncMock(), + ) + return _PROXY_LiteLLMManagedFiles( + internal_usage_cache=DualCache(), + prisma_client=MagicMock(db=MagicMock(litellm_managedfiletable=table)), + ), table + + +@pytest.mark.asyncio +async def test_afile_delete_bedrock_uses_deployment_bucket_and_signed_s3_delete(monkeypatch): + import httpx + import respx + + from litellm import Router + + monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) + monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + router = Router( + model_list=[ + { + "model_name": "bedrock-batch", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "secret", + "aws_region_name": "us-west-2", + "s3_bucket_name": "my-bucket", + }, + "model_info": {"id": "model-123"}, + } + ], + num_retries=0, + ) + s3_uri = "s3://my-bucket/litellm-bedrock-files/input.jsonl" + unified_file_id = _managed_deletion_file_id(s3_uri) + managed_files, table = _managed_files_with_deletion_row(unified_file_id, s3_uri, None) + with respx.mock: + route = respx.delete( + "https://s3.us-west-2.amazonaws.com/my-bucket/litellm-bedrock-files/input.jsonl" + ).mock(return_value=httpx.Response(204)) + response = await managed_files.afile_delete( + file_id=unified_file_id, + litellm_parent_otel_span=None, + llm_router=router, + _litellm_internal_model_credentials={"s3_bucket_name": "request-bucket"}, + ) + + assert len(route.calls) == 1 + assert route.calls[0].request.headers["Authorization"].startswith("AWS4-HMAC-SHA256") + assert response.id == unified_file_id + assert response.deleted is True + table.delete.assert_awaited_once_with(where={"unified_file_id": unified_file_id}) + + +@pytest.mark.asyncio +async def test_afile_delete_returns_managed_id_for_stored_provider_output(): + from openai.types import FileDeleted + + provider_file_id = "file-error-output" + unified_file_id = _managed_deletion_file_id(provider_file_id) + stored_file = _make_file_object(provider_file_id) + managed_files, table = _managed_files_with_deletion_row(unified_file_id, provider_file_id, stored_file) + router = MagicMock( + get_deployment_credentials_with_provider=MagicMock(return_value=None), + afile_delete=AsyncMock(return_value=FileDeleted(id=provider_file_id, object="file", deleted=True)), + ) + response = await managed_files.afile_delete( + file_id=unified_file_id, + litellm_parent_otel_span=None, + llm_router=router, + _litellm_internal_model_credentials={"s3_bucket_name": "request-bucket"}, + ) + + assert response.id == unified_file_id + assert response.object == "file" + assert response.filename == stored_file.filename + assert stored_file.id == provider_file_id + router.afile_delete.assert_awaited_once_with(model="model-123", file_id=provider_file_id) + table.delete.assert_awaited_once_with(where={"unified_file_id": unified_file_id}) + + @pytest.mark.asyncio async def test_afile_content_bedrock_unified_id_end_to_end(monkeypatch): """ From 1d7e81cf5d3a29dd4731b3282cf0842aac854ea1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:47:06 -0700 Subject: [PATCH 069/411] fix(streaming): guard empty choices and missing role when assembling stream chunks --- .../streaming_chunk_builder_utils.py | 23 ++-- .../test_streaming_chunk_builder_utils.py | 129 +++++++----------- 2 files changed, 66 insertions(+), 86 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 81955fe769e..cf9604a0fd5 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -24,6 +24,7 @@ from litellm.types.utils import ( Choices, CompletionTokensDetails, CompletionTokensDetailsWrapper, + Delta, Function, FunctionCall, ModelResponse, @@ -326,6 +327,18 @@ class ChunkProcessor: return chunk_id return "" + @staticmethod + def _get_role_from_chunks(chunks: Sequence["_BaseChunk"]) -> str: + return ChunkProcessor._role_of_choice(next((c["choices"][0] for c in chunks if c.get("choices")), None)) + + @staticmethod + def _role_of_choice(choice: object) -> str: + match choice: + case StreamingChoices(delta=Delta(role=str() as role)) | {"delta": {"role": str() as role}} if role: + return role + case _: + return "assistant" + @staticmethod def _get_model_from_chunks(chunks: Sequence["_BaseChunk"], first_chunk_model: str) -> str: """ @@ -353,15 +366,7 @@ class ChunkProcessor: model: Final = ChunkProcessor._get_model_from_chunks(chunks, first_chunk_model) system_fingerprint: Final = chunk.get("system_fingerprint", None) - # Fall back to None rather than `chunk`: if no chunk carries a non-empty - # `choices` array, indexing [0] on the first chunk raises IndexError. - first_chunk_with_choices = next((c for c in chunks if c.get("choices")), None) - role: str = "assistant" - if first_chunk_with_choices is not None: - _choices = first_chunk_with_choices["choices"] - if len(_choices) > 0: - # `delta` may be absent or omit `role` (e.g. content-only deltas). - role = _choices[0].get("delta", {}).get("role") or "assistant" + role: Final = ChunkProcessor._get_role_from_chunks(chunks) finish_reason = "stop" for chunk in chunks: if "choices" in chunk and len(chunk["choices"]) > 0: diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 2d2451e73f7..626b8a63b20 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1,4 +1,6 @@ import json +from collections.abc import Mapping, Sequence +from typing import Final import pytest @@ -1478,104 +1480,77 @@ def test_calculate_usage_fills_unknown_split_from_reasoning_estimate( assert usage.completion_tokens_details.text_tokens == expected_text_tokens -def _empty_choices_chunk(**extra): - chunk = { - "id": "chatcmpl-empty-choices", +def _openai_chunk( + choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None +) -> dict[str, object]: + base: Final = { + "id": "chatcmpl-lit6552", "object": "chat.completion.chunk", "created": 1, - "model": "claude-opus-4-8", - "choices": [], + "model": "gpt-5.4-mini", + "choices": list(choices), } - chunk.update(extra) - return chunk + return base if usage is None else {**base, "usage": dict(usage)} @pytest.mark.parametrize( "chunks", [ + pytest.param([_openai_chunk(choices=[]), _openai_chunk(choices=[])], id="all_empty_choices_dicts"), pytest.param( - [_empty_choices_chunk(), _empty_choices_chunk()], - id="all_chunks_have_empty_choices", - ), - pytest.param( - [ - _empty_choices_chunk(usage={"prompt_tokens": 10}), - _empty_choices_chunk(usage={"completion_tokens": 0}), - ], - id="usage_only_chunks", + [ModelResponseStream(model="gpt-5.4-mini", choices=[]) for _ in range(2)], + id="all_empty_choices_objects", ), ], ) -def test_build_base_response_handles_empty_choices(chunks): - """Empty `choices` arrays must not raise IndexError. - - `next((c for c in chunks if c.get("choices")), chunk)` used to fall back to the - first chunk, whose `choices` may be `[]`, so `["choices"][0]` went out of range. - The resulting error is surfaced to the client mid-stream and the request never - reaches SpendLogs. - """ - processor = ChunkProcessor(chunks=list(chunks)) - - response = processor.build_base_response(list(chunks)) +def test_stream_chunk_builder_survives_all_empty_choices(chunks: Sequence[object]) -> None: + response: Final = stream_chunk_builder(chunks=list(chunks)) + assert response is not None assert response.choices[0].message.role == "assistant" + assert response.choices[0].finish_reason == "stop" + + +def test_stream_chunk_builder_keeps_usage_from_usage_only_frames() -> None: + usage_frame: Final = _openai_chunk( + choices=[], usage={"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10} + ) + + response: Final = stream_chunk_builder(chunks=[usage_frame]) + + assert response is not None + assert response.choices[0].message.role == "assistant" + assert response.usage.prompt_tokens == 10 + assert response.usage.total_tokens == 10 @pytest.mark.parametrize( "delta", - [ - pytest.param({"content": "Hello"}, id="delta_without_role"), - pytest.param({}, id="delta_empty_dict"), - ], + [pytest.param({"content": "Hi"}, id="delta_without_role"), pytest.param({}, id="empty_delta")], ) -def test_build_base_response_handles_delta_without_role(delta): - """A `delta` that omits `role` must not raise KeyError.""" - chunks = [ - { - "id": "chatcmpl-no-role", - "object": "chat.completion.chunk", - "created": 1, - "model": "claude-opus-4-8", - "choices": [{"index": 0, "delta": delta, "finish_reason": None}], - } +def test_stream_chunk_builder_defaults_role_when_delta_omits_it(delta: Mapping[str, str]) -> None: + chunks: Final = [ + _openai_chunk(choices=[{"index": 0, "delta": dict(delta), "finish_reason": None}]), + _openai_chunk(choices=[{"index": 0, "delta": {"content": "!"}, "finish_reason": "stop"}]), ] - processor = ChunkProcessor(chunks=list(chunks)) - response = processor.build_base_response(list(chunks)) - - assert response.choices[0].message.role == "assistant" - - -def test_build_base_response_still_reads_role_and_finish_reason(): - """Regression guard: well-formed chunks keep their role and finish_reason.""" - chunks = [ - _empty_choices_chunk(), - { - "id": "chatcmpl-normal", - "object": "chat.completion.chunk", - "created": 1, - "model": "claude-opus-4-8", - "choices": [ - { - "index": 0, - "delta": {"role": "assistant", "content": "Hi"}, - "finish_reason": None, - } - ], - }, - { - "id": "chatcmpl-normal", - "object": "chat.completion.chunk", - "created": 2, - "model": "claude-opus-4-8", - "choices": [ - {"index": 0, "delta": {"content": "!"}, "finish_reason": "stop"} - ], - }, - ] - processor = ChunkProcessor(chunks=list(chunks)) - - response = processor.build_base_response(list(chunks)) + response: Final = stream_chunk_builder(chunks=chunks) + assert response is not None assert response.choices[0].message.role == "assistant" + assert response.choices[0].message.content == delta.get("content", "") + "!" assert response.choices[0].finish_reason == "stop" + + +def test_stream_chunk_builder_reads_role_from_first_frame_with_choices() -> None: + chunks: Final = [ + _openai_chunk(choices=[]), + _openai_chunk(choices=[{"index": 0, "delta": {"role": "user", "content": "Hi"}, "finish_reason": None}]), + _openai_chunk(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}]), + ] + + response: Final = stream_chunk_builder(chunks=chunks) + + assert response is not None + assert response.choices[0].message.role == "user" + assert response.choices[0].message.content == "Hi" From 6f99917b332a1038ac83f5f5a5f8614fa573b1e6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:57:54 -0700 Subject: [PATCH 070/411] fix(router): rewrite multi-segment model groups as whole passthrough path segments --- .../base_llm/passthrough/transformation.py | 4 +- tests/test_litellm/test_router.py | 46 ++++++++++++------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index 464c121ecc1..2c21df01b5b 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -1,3 +1,4 @@ +import re from abc import abstractmethod from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Optional, Union @@ -27,7 +28,8 @@ def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> def replace_path_segment(endpoint: str, segment: str, replacement: str) -> str: - return "/".join(replacement if part == segment else part for part in endpoint.split("/")) + bounded_segment: Final = re.compile(rf"(? Date: Mon, 7 Sep 2026 18:00:48 -0700 Subject: [PATCH 071/411] fix(azure): let the caller's api-version win over the deployment's on passthrough relays --- litellm/llms/azure/passthrough/transformation.py | 4 ++-- .../test_azure_passthrough_transformation.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 983033a8b14..062c60d7231 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -61,11 +61,11 @@ class AzurePassthroughConfig(BasePassthroughConfig): routed_endpoint: Final = replace_path_segment(endpoint, model_group, model) if model_group else endpoint native_endpoint: Final = strip_leading_model_segment(routed_endpoint, (model,)) + caller_api_version: Final = request_query_params.get("api-version") if request_query_params else None complete_url: Final = BaseAzureLLM._get_base_azure_url( api_base=base_target_url, - litellm_params=litellm_params, + litellm_params={**litellm_params, "api_version": caller_api_version or litellm_params.get("api_version")}, route=native_endpoint, - default_api_version=request_query_params.get("api-version") if request_query_params else None, ) return ( httpx.URL(complete_url), diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 8ffd632fe85..262a281aeb3 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -207,18 +207,24 @@ def _complete_url(request_query_params: dict, litellm_params: dict) -> httpx.URL return url -def test_azure_passthrough_url_falls_back_to_the_callers_api_version(): +def test_azure_passthrough_url_forwards_the_callers_api_version(): url = _complete_url(request_query_params={"api-version": "2025-04-01-preview"}, litellm_params={}) assert url.path == "/openai/deployments/gpt-4.1-mini/chat/completions" assert url.params["api-version"] == "2025-04-01-preview" -def test_azure_passthrough_url_prefers_the_deployments_api_version(): +def test_azure_passthrough_url_prefers_the_callers_api_version_over_the_deployments(): url = _complete_url( request_query_params={"api-version": "2025-04-01-preview"}, litellm_params={"api_version": "2024-10-21"} ) + assert url.params["api-version"] == "2025-04-01-preview" + + +def test_azure_passthrough_url_fills_in_the_deployments_api_version_when_the_caller_sends_none(): + url = _complete_url(request_query_params={}, litellm_params={"api_version": "2024-10-21"}) + assert url.params["api-version"] == "2024-10-21" From 170fece7db9b4997fab48cd716811975e43f11f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:08:25 -0700 Subject: [PATCH 072/411] fix(token_counter): release the GIL for HuggingFace counts and cap exact counting per string Both proxy token counting endpoints already count in a worker thread, but the HuggingFace tokenizer's encode holds the GIL for the whole call, so a 600k-token count on a Claude model still froze the event loop for up to 0.8 s and every other request with it. Count through encode_batch_fast, which releases the GIL, and tokenize at most TOKEN_COUNTER_MAX_EXACT_CHARS characters of any one string (default 4,000,000), scaling the exact count of that prefix by the string's length above it so the largest payloads stay bounded. --- litellm/constants.py | 6 ++ litellm/litellm_core_utils/token_counter.py | 25 ++++++- .../litellm_core_utils/test_token_counter.py | 71 ++++++++++++++++++- tests/test_litellm/proxy/test_proxy_server.py | 34 ++++++++- 4 files changed, 132 insertions(+), 4 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index d53686e5e5b..a9112c90a54 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -391,6 +391,12 @@ TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS: Final = get_env_int_in_range( minimum=1, maximum=TIKTOKEN_ENCODE_MAX_CHUNK_SIZE_CHARS, ) +TOKEN_COUNTER_MAX_EXACT_CHARS: Final = get_env_int_in_range( + "TOKEN_COUNTER_MAX_EXACT_CHARS", + default=4_000_000, + minimum=1, + maximum=1_000_000_000, +) MAX_TILE_WIDTH: Final = int(os.getenv("MAX_TILE_WIDTH", 512)) MAX_TILE_HEIGHT: Final = int(os.getenv("MAX_TILE_HEIGHT", 512)) OPENAI_FILE_SEARCH_COST_PER_1K_CALLS: Final = float(os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000)) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 3732ffd734c..d526f0f2996 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -8,6 +8,7 @@ from typing import Final, Literal, cast import httpx import tiktoken +from tokenizers import Tokenizer import litellm from litellm import verbose_logger @@ -21,6 +22,7 @@ from litellm.constants import ( MAX_TILE_HEIGHT, MAX_TILE_WIDTH, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, + TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.default_encoding import encoding as default_encoding from litellm.litellm_core_utils.url_utils import safe_get @@ -538,9 +540,28 @@ def _count_extra( return num_tokens +def _get_extrapolating_count_function( + count_exactly: TokenCounterFunction, + max_exact_chars: int = TOKEN_COUNTER_MAX_EXACT_CHARS, +) -> TokenCounterFunction: + def count_tokens(text: str) -> int: + if len(text) <= max_exact_chars: + return count_exactly(text) + return round(count_exactly(text[:max_exact_chars]) * len(text) / max_exact_chars) + + return count_tokens + + def _get_count_function( model: str | None, custom_tokenizer: dict | SelectTokenizerResponse | None = None, +) -> TokenCounterFunction: + return _get_extrapolating_count_function(_get_exact_count_function(model, custom_tokenizer)) + + +def _get_exact_count_function( + model: str | None, + custom_tokenizer: dict | SelectTokenizerResponse | None = None, ) -> TokenCounterFunction: """ Get the function to count tokens based on the model and custom tokenizer.""" @@ -549,10 +570,10 @@ def _get_count_function( if model is not None or custom_tokenizer is not None: tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model) if tokenizer_json["type"] == "huggingface_tokenizer": + tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: - enc: Final = tokenizer_json["tokenizer"].encode(text) - return len(enc.ids) + return len(tokenizer.encode_batch_fast([text])[0]) return count_tokens elif tokenizer_json["type"] == "openai_tokenizer": diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 4694fa8fbed..fddeacc37f9 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1,8 +1,10 @@ #### What this tests #### # This tests litellm.token_counter.token_counter() function +import asyncio import importlib import time import traceback +from typing import Final from unittest.mock import MagicMock import pytest @@ -14,7 +16,12 @@ import litellm from litellm import create_pretrained_tokenizer, decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.litellm_core_utils.token_counter import _get_tiktoken_count_function +from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.token_counter import ( + _get_exact_count_function, + _get_extrapolating_count_function, + _get_tiktoken_count_function, +) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new from tests.large_text import text from tests.test_litellm.litellm_core_utils.messages_with_counts import ( @@ -120,6 +127,68 @@ def test_valid_chunk_size_config_is_honoured(monkeypatch): importlib.reload(litellm.constants) +async def _loop_wake_lags(until: asyncio.Event) -> tuple[float, ...]: + async def wake_lag() -> float: + started: Final = time.perf_counter() + await asyncio.sleep(0.001) + return time.perf_counter() - started - 0.001 + + return tuple([await wake_lag() for _ in iter(until.is_set, True)]) + + +async def test_huggingface_count_in_a_worker_thread_leaves_the_event_loop_free(): + counted: Final = asyncio.Event() + + async def count_off_the_loop() -> tuple[int, float]: + started: Final = time.perf_counter() + try: + tokens: Final = await asyncify(token_counter_new)(model="claude-fable-5", text=text * 100) + return tokens, time.perf_counter() - started + finally: + counted.set() + + (tokens, took), lags = await asyncio.gather(count_off_the_loop(), _loop_wake_lags(counted)) + + assert tokens > 0 + assert max(lags) < took / 4, f"the event loop stalled {max(lags):.3f}s during a {took:.3f}s count" + + +@pytest.mark.parametrize( + ("max_exact_chars", "expected"), + [(1_000, 10_000), (5_000, 6_000), (10_000, 6_000)], +) +def test_count_above_the_cap_scales_the_exact_count_of_the_prefix(max_exact_chars, expected): + def count_exactly(chunk: str) -> int: + return chunk.count("a") + len(chunk) + + count_tokens: Final = _get_extrapolating_count_function(count_exactly, max_exact_chars=max_exact_chars) + + assert count_tokens("a" * 1_000 + "b" * 4_000) == expected + + +def test_token_counter_applies_the_default_cap(): + max_exact_chars: Final = litellm.constants.TOKEN_COUNTER_MAX_EXACT_CHARS + prefix: Final = ("The quick brown fox jumps over the lazy dog. " * (max_exact_chars // 45 + 1))[:max_exact_chars] + over_the_cap: Final = prefix + "a" * 200_000 + scaled: Final = round(token_counter_new(model="gpt-5.6", text=prefix) * len(over_the_cap) / max_exact_chars) + + assert token_counter_new(model="gpt-5.6", text=over_the_cap) == scaled + assert _get_exact_count_function("gpt-5.6")(over_the_cap) != scaled + + +@pytest.mark.parametrize( + ("configured", "expected"), + [("2048", 2048), ("0", 4_000_000), ("not-an-int", 4_000_000)], +) +def test_max_exact_chars_config_is_honoured(monkeypatch, configured, expected): + monkeypatch.setenv("TOKEN_COUNTER_MAX_EXACT_CHARS", configured) + try: + assert importlib.reload(litellm.constants).TOKEN_COUNTER_MAX_EXACT_CHARS == expected + finally: + monkeypatch.delenv("TOKEN_COUNTER_MAX_EXACT_CHARS") + importlib.reload(litellm.constants) + + def test_token_counter_with_prefix(): messages = [ {"role": "user", "content": "Who won the world cup in 2022?"}, diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index b7bb58378d4..20b41cf1530 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -6,6 +6,7 @@ import os import re import socket import subprocess +import time import types from datetime import datetime, timedelta, timezone from pathlib import Path @@ -28,7 +29,7 @@ from litellm.caching.caching import RedisCache from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded from litellm.caching.dual_cache import DualCache -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, TokenCountRequest, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.proxy_server import app, initialize from litellm.utils import _invalidate_model_cost_lowercase_map @@ -12857,3 +12858,34 @@ async def test_update_general_settings_keeps_yaml_openai_websocket_passthrough() import litellm.proxy.proxy_server as ps assert ps.general_settings["enable_openai_websocket_passthrough"] is False + + +async def _loop_wake_lags(until: asyncio.Event) -> tuple[float, ...]: + async def wake_lag() -> float: + started: Final = time.perf_counter() + await asyncio.sleep(0.001) + return time.perf_counter() - started - 0.001 + + return tuple([await wake_lag() for _ in iter(until.is_set, True)]) + + +async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_count(monkeypatch): + from tests.large_text import text + + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + counted: Final = asyncio.Event() + + async def count_off_the_loop() -> tuple[int, float]: + started: Final = time.perf_counter() + try: + response: Final = await proxy_server_module.token_counter( + TokenCountRequest(model="claude-fable-5", prompt=text * 100) + ) + return response.total_tokens, time.perf_counter() - started + finally: + counted.set() + + (tokens, took), lags = await asyncio.gather(count_off_the_loop(), _loop_wake_lags(counted)) + + assert tokens > 0 + assert max(lags) < took / 4, f"the event loop stalled {max(lags):.3f}s during a {took:.3f}s count" From 17b70035924b7bb2e97065a7f3512c540e105d66 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:11:17 -0700 Subject: [PATCH 073/411] fix(azure_ai): cost embeddings, responses, images, and rerank relays instead of logging zero --- .../llms/azure/passthrough/transformation.py | 45 ++++++- .../azure_ai/passthrough/transformation.py | 32 ++++- .../base_llm/passthrough/transformation.py | 20 ++- .../test_azure_passthrough_transformation.py | 120 +++++++++++++++--- ...est_azure_ai_passthrough_transformation.py | 68 +++++++++- 5 files changed, 261 insertions(+), 24 deletions(-) diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 062c60d7231..d7078d94a0b 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -1,4 +1,5 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Optional import httpx @@ -9,12 +10,14 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.base_llm.passthrough.transformation import ( BasePassthroughConfig, + relayed_json_object, replace_path_segment, strip_leading_model_segment, ) from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse if TYPE_CHECKING: from httpx import URL @@ -38,6 +41,40 @@ def _relayed_messages(litellm_logging_obj: Logging) -> Sequence[Mapping[str, obj return details.request_data.messages if details.request_data else None +@dataclass(frozen=True, slots=True) +class OpenAIRelayShape: + path_suffix: str + call_type: CallTypes + parse: Callable[[Mapping[str, object]], EmbeddingResponse | ImageResponse | ResponsesAPIResponse] + + +OPENAI_RELAY_SHAPES: Final = ( + OpenAIRelayShape("/embeddings", CallTypes.aembedding, EmbeddingResponse.model_validate), + OpenAIRelayShape("/responses", CallTypes.aresponses, ResponsesAPIResponse.model_validate), + OpenAIRelayShape("/images/generations", CallTypes.aimage_generation, ImageResponse.model_validate), +) + + +def logged_openai_response( + httpx_response: Response, logging_obj: Logging, endpoint: str +) -> EmbeddingResponse | ImageResponse | ResponsesAPIResponse | None: + relayed_path: Final = f"/{endpoint.strip('/')}" + shape: Final = next( + (candidate for candidate in OPENAI_RELAY_SHAPES if relayed_path.endswith(candidate.path_suffix)), None + ) + body: Final = relayed_json_object(httpx_response) if shape else None + if shape is None or body is None: + return None + try: + parsed: Final = shape.parse(body) + except ValidationError: + return None + logging_obj.call_type = ( + shape.call_type.value + ) # rebind-ok: routes cost calculation to the relayed shape's pricing path + return parsed + + class AzurePassthroughConfig(BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: return bool(request_data.get("stream")) @@ -114,13 +151,13 @@ class AzurePassthroughConfig(BasePassthroughConfig): request_data: dict, logging_obj: Logging, endpoint: str, - ) -> Optional["CostResponseTypes"]: + ) -> Optional["CostResponseTypes | ResponsesAPIResponse"]: from litellm import encoding from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.utils import ModelResponse if "chat/completions" not in endpoint: - return None + return logged_openai_response(httpx_response, logging_obj, endpoint) openai_chat_config: Final = OpenAIGPTConfig() diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index 4cb93b4f8f9..c49e10566ed 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -14,8 +14,13 @@ from litellm.llms.azure_ai.common_utils import ( get_azure_ai_auth_headers, ) from litellm.llms.azure_ai.ocr.common_utils import get_azure_ai_ocr_config -from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, strip_leading_model_segment +from litellm.llms.base_llm.passthrough.transformation import ( + BasePassthroughConfig, + relayed_json_object, + strip_leading_model_segment, +) from litellm.types.llms.openai import AllMessageValues +from litellm.types.rerank import RerankResponse from litellm.types.utils import CallTypes, StandardPassThroughResponseObject if TYPE_CHECKING: @@ -23,6 +28,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse + from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import CostResponseTypes @@ -77,6 +83,18 @@ def relayed_body(httpx_response: Response) -> str | dict: return body if isinstance(body, dict) else httpx_response.text +def logged_rerank_response(httpx_response: Response, logging_obj: Logging, endpoint: str) -> RerankResponse | None: + body: Final = relayed_json_object(httpx_response) if f"/{endpoint.strip('/')}".endswith("/rerank") else None + if body is None: + return None + try: + rerank_response: Final = RerankResponse.model_validate(body) + except ValidationError: + return None + logging_obj.call_type = CallTypes.arerank.value # rebind-ok: routes cost calculation to the per-query rerank path + return rerank_response + + class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): def __init__(self, ocr_config_for: Callable[[str], BaseOCRConfig | None] = get_azure_ai_ocr_config) -> None: super().__init__() @@ -130,7 +148,14 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): request_data: Mapping[str, object], logging_obj: Logging, endpoint: str, - ) -> CostResponseTypes | OCRResponse | StandardPassThroughResponseObject | None: + ) -> ( + CostResponseTypes + | OCRResponse + | RerankResponse + | ResponsesAPIResponse + | StandardPassThroughResponseObject + | None + ): from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig chat_result: Final = AzurePassthroughConfig().logging_non_streaming_response( # pyright: ignore[reportUnknownMemberType] # the Azure config still types request_data as a bare dict @@ -146,6 +171,9 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): ocr_result: Final = self.logged_ocr_response(model, httpx_response, logging_obj, endpoint) if ocr_result is not None: return ocr_result + rerank_result: Final = logged_rerank_response(httpx_response, logging_obj, endpoint) + if rerank_result is not None: + return rerank_result return StandardPassThroughResponseObject(response=relayed_body(httpx_response)) def logged_ocr_response( diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index 2c21df01b5b..975b74137db 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -3,18 +3,25 @@ from abc import abstractmethod from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Optional, Union +from pydantic import TypeAdapter, ValidationError + from ..base_utils import BaseLLMModelInfo if TYPE_CHECKING: from httpx import URL, Headers, Response from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.rerank import RerankResponse from litellm.types.utils import CostResponseTypes, StandardPassThroughResponseObject from ..chat.transformation import BaseLLMException from ..ocr.transformation import OCRResponse +RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) + + def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> str: path: Final = endpoint.lstrip("/") for model_name in model_names: @@ -32,6 +39,15 @@ def replace_path_segment(endpoint: str, segment: str, replacement: str) -> str: return bounded_segment.sub(lambda _: replacement, endpoint) +def relayed_json_object(httpx_response: "Response") -> Mapping[str, object] | None: + if httpx_response.status_code != 200: + return None + try: + return RELAYED_JSON_OBJECT.validate_python(httpx_response.json()) + except (ValueError, ValidationError): + return None + + class BasePassthroughConfig(BaseLLMModelInfo): @abstractmethod def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: @@ -123,7 +139,9 @@ class BasePassthroughConfig(BaseLLMModelInfo): request_data: dict, logging_obj: "LiteLLMLoggingObj", endpoint: str, - ) -> Optional["CostResponseTypes | OCRResponse | StandardPassThroughResponseObject"]: + ) -> Optional[ + "CostResponseTypes | OCRResponse | RerankResponse | ResponsesAPIResponse | StandardPassThroughResponseObject" + ]: pass def handle_logging_collected_chunks( diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 262a281aeb3..2723a13c3fc 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -1,15 +1,16 @@ import json +from datetime import datetime from unittest.mock import MagicMock import httpx import pytest import litellm - - +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig -from litellm.types.utils import ModelResponse +from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import EmbeddingResponse, ModelResponse def _azure_chat_completion_body(): @@ -77,25 +78,112 @@ def test_azure_passthrough_logging_non_streaming_response_chat_completions(): assert result.usage.total_tokens == 18 -def test_azure_passthrough_logging_non_streaming_response_unknown_endpoint_returns_none(): - """ - Endpoints other than chat/completions (responses, messages, images) fall - through to None — matches base-class behavior and Bedrock's "unknown - endpoint" handling. Not a regression; just scoping. - """ - config = AzurePassthroughConfig() - logging_obj = MagicMock() - - result = config.logging_non_streaming_response( - model="gpt-4.1-mini", +def _relay_logging_obj(model: str) -> Logging: + logging_obj = Logging( + model=model, + messages=[], + stream=False, + call_type="allm_passthrough_route", + start_time=datetime.now(), + litellm_call_id="call-1", + function_id="fn-1", + ) + logging_obj.update_environment_variables( + model=model, + litellm_params={"api_base": "https://my-resource.openai.azure.com", "custom_llm_provider": "azure"}, + optional_params={}, custom_llm_provider="azure", - httpx_response=_make_httpx_response(_azure_chat_completion_body()), + ) + return logging_obj + + +def _relay_logging_result(model: str, endpoint: str, body, status_code: int = 200): + logging_obj = _relay_logging_obj(model) + response = httpx.Response( + status_code=status_code, + headers={"content-type": "application/json"}, + content=json.dumps(body).encode("utf-8"), + request=httpx.Request("POST", f"https://my-resource.openai.azure.com/{endpoint}?api-version=2025-04-01-preview"), + ) + result = AzurePassthroughConfig().logging_non_streaming_response( + model=model, + custom_llm_provider="azure", + httpx_response=response, request_data={}, logging_obj=logging_obj, - endpoint="openai/responses", + endpoint=endpoint, + ) + return result, logging_obj + + +EMBEDDINGS_BODY = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1000, "total_tokens": 1000}, +} + +RESPONSES_BODY = { + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4.1-mini", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + "usage": {"input_tokens": 1000, "output_tokens": 100, "total_tokens": 1100}, +} + + +def test_azure_passthrough_embeddings_relay_is_costed_per_input_token(): + result, logging_obj = _relay_logging_result( + "text-embedding-3-small", "openai/deployments/text-embedding-3-small/embeddings", EMBEDDINGS_BODY + ) + per_token = litellm.get_model_info("azure/text-embedding-3-small")["input_cost_per_token"] + + assert isinstance(result, EmbeddingResponse) + assert logging_obj.call_type == "aembedding" + assert per_token > 0 + assert logging_obj._response_cost_calculator(result=result) == pytest.approx(1000 * per_token) + + +def test_azure_passthrough_responses_relay_is_costed_per_token(): + result, logging_obj = _relay_logging_result("gpt-4.1-mini", "openai/responses", RESPONSES_BODY) + info = litellm.get_model_info("azure/gpt-4.1-mini") + + assert isinstance(result, ResponsesAPIResponse) + assert logging_obj.call_type == "aresponses" + assert logging_obj._response_cost_calculator(result=result) == pytest.approx( + 1000 * info["input_cost_per_token"] + 100 * info["output_cost_per_token"] + ) + + +def test_azure_passthrough_failed_embeddings_relay_is_not_costed(): + result, logging_obj = _relay_logging_result( + "text-embedding-3-small", + "openai/deployments/text-embedding-3-small/embeddings", + {"error": {"code": "429", "message": "rate limited"}}, + status_code=429, ) assert result is None + assert logging_obj.call_type == "allm_passthrough_route" + + +def test_azure_passthrough_logging_non_streaming_response_unknown_endpoint_returns_none(): + result, logging_obj = _relay_logging_result( + "gpt-4o-mini-tts", "openai/deployments/gpt-4o-mini-tts/audio/speech", {"audio": "..."} + ) + + assert result is None + assert logging_obj.call_type == "allm_passthrough_route" def _sse_line(payload: dict) -> str: diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index 644316c2e37..788aced50f8 100644 --- a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -9,7 +9,8 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.passthrough.transformation import AzureAIPassthroughConfig from litellm.llms.base_llm.ocr.transformation import OCRResponse -from litellm.types.utils import LlmProviders, ModelResponse +from litellm.types.rerank import RerankResponse +from litellm.types.utils import EmbeddingResponse, ImageResponse, LlmProviders, ModelResponse from litellm.utils import ProviderConfigManager FOUNDRY_BASE = "https://my-resource.services.ai.azure.com" @@ -370,6 +371,71 @@ def test_unparseable_ocr_body_falls_back_to_the_passthrough_object(): assert logging_obj.call_type == "allm_passthrough_route" +EMBEDDINGS_BODY = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "embed-v-4-0", + "usage": {"prompt_tokens": 1200, "total_tokens": 1200}, +} + +RERANK_BODY = { + "id": "rerank-1", + "results": [{"index": 1, "relevance_score": 0.9}, {"index": 0, "relevance_score": 0.2}], + "meta": {"api_version": {"version": "2"}, "billed_units": {"search_units": 2}}, +} + +IMAGE_BODY = {"created": 1, "data": [{"b64_json": "AAAA"}]} + + +def test_foundry_embeddings_relay_is_costed_per_input_token(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), "embed-v-4-0", "models/embeddings", EMBEDDINGS_BODY + ) + per_token = litellm.get_model_info("azure_ai/embed-v-4-0")["input_cost_per_token"] + + assert isinstance(result, EmbeddingResponse) + assert logging_obj.call_type == "aembedding" + assert per_token > 0 + assert logging_obj._response_cost_calculator(result=result) == pytest.approx(1200 * per_token) + + +def test_cohere_rerank_relay_is_costed_per_search_unit(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), "cohere-rerank-v4.0-fast", "providers/cohere/v2/rerank", RERANK_BODY + ) + per_query = litellm.get_model_info("azure_ai/cohere-rerank-v4.0-fast")["input_cost_per_query"] + + assert isinstance(result, RerankResponse) + assert logging_obj.call_type == "arerank" + assert per_query > 0 + assert logging_obj._response_cost_calculator(result=result) == pytest.approx(2 * per_query) + + +def test_image_generation_relay_is_costed_per_image(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), "FLUX.2-pro", "openai/deployments/FLUX.2-pro/images/generations", IMAGE_BODY + ) + per_image = litellm.get_model_info("azure_ai/FLUX.2-pro")["output_cost_per_image"] + + assert isinstance(result, ImageResponse) + assert logging_obj.call_type == "aimage_generation" + assert per_image > 0 + assert logging_obj._response_cost_calculator(result=result) == pytest.approx(per_image) + + +def test_rejected_rerank_relay_keeps_the_passthrough_object_and_call_type(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), + "cohere-rerank-v4.0-fast", + "providers/cohere/v2/rerank", + {"message": "invalid request"}, + status_code=400, + ) + + assert result == {"response": {"message": "invalid request"}} + assert logging_obj.call_type == "allm_passthrough_route" + + def test_streaming_chat_completion_chunks_are_costed_like_azure(): head = {"id": "chatcmpl-1", "object": "chat.completion.chunk", "created": 1, "model": "gpt-5.4-mini"} chunks = [ From 3b199cd3da3fc97a6a373a1247fb798fa2353ca1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:11:58 -0700 Subject: [PATCH 074/411] fix(azure_ai): price seven Foundry catalog names and charge the model router fee once Add cost map entries for azure_ai/gpt-chat-latest, codex-mini, whisper, model-router, cohere-command-a, grok-4-20-reasoning, and grok-4-20-non-reasoning, priced from the live Azure AI Foundry and Azure OpenAI pricing pages and the Azure Retail Prices API. Skip the model router flat fee when the response model is the router entry itself, since the generic cost already priced that fee. Before, azure_ai/model_router charged it twice. Resolves LIT-3157 --- litellm/llms/azure_ai/cost_calculator.py | 76 +++----- ...odel_prices_and_context_window_backup.json | 130 +++++++++++++ model_prices_and_context_window.json | 130 +++++++++++++ .../azure_ai/test_azure_ai_cost_calculator.py | 26 +++ ...azure_ai_foundry_catalog_model_metadata.py | 176 ++++++++++++++++++ 5 files changed, 492 insertions(+), 46 deletions(-) create mode 100644 tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 95f536296a2..141148f06e7 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -56,6 +56,27 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl return 0.0 +ROUTER_FEE_ENTRY_NAMES: Final = frozenset({"model-router", "model_router"}) + + +def _prices_router_fee_itself(model: str) -> bool: + return model.lower().rsplit("/", 1)[-1] in ROUTER_FEE_ENTRY_NAMES + + +def _base_cost_per_token(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float] | None: + try: + return generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier + ) + except Exception as e: + if not _is_azure_model_router(model): + raise + verbose_logger.debug( + "Azure AI Model Router: model '%s' not in cost map, calculating routing flat cost only. Error: %s", model, e + ) + return None + + def cost_per_token( model: str, usage: Usage, @@ -66,9 +87,9 @@ def cost_per_token( """ Calculate the cost per token for Azure AI models. - For Azure AI Foundry Model Router: - - Adds a flat cost of $0.14 per million input tokens (from model_prices_and_context_window.json) - - Plus the cost of the actual model used (handled by generic_cost_per_token) + For Azure AI Foundry Model Router the routing fee (the azure_ai/model_router entry, $0.14 per + million input tokens) is added on top of the routed model's cost. When the response model is + the router entry itself, generic_cost_per_token has already charged that fee. Args: model: str, the model name without provider prefix (from response) @@ -83,49 +104,12 @@ def cost_per_token( ValueError: If the model is not found in the cost map and cost cannot be calculated (except for Model Router models where we return just the routing flat cost) """ - prompt_cost = 0.0 - completion_cost = 0.0 - - # Determine if this was a model router request - # Check both the response model and the request model is_router_request: Final = _is_azure_model_router(model) or ( request_model is not None and _is_azure_model_router(request_model) ) - - # Calculate base cost using generic cost calculator - # This may raise an exception if the model is not in the cost map - try: - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=usage, - custom_llm_provider="azure_ai", - service_tier=service_tier, - ) - except Exception as e: - # For Model Router, the model name (e.g., "azure-model-router") may not be in the cost map - # because it's a routing service, not an actual model. In this case, we continue - # to calculate just the routing flat cost. - if not _is_azure_model_router(model): - # Re-raise for non-router models - they should have pricing defined - raise - verbose_logger.debug( - "Azure AI Model Router: model '%s' not in cost map, calculating routing flat cost only. Error: %s", model, e - ) - - # Add flat cost for Azure Model Router - # The flat cost is defined in model_prices_and_context_window.json for azure_ai/model_router - if is_router_request: - # Use the request model for flat cost calculation if available, otherwise use response model - router_model_for_calc: Final = request_model if request_model else model - router_flat_cost: Final = calculate_azure_model_router_flat_cost(router_model_for_calc, usage.prompt_tokens) - - if router_flat_cost > 0: - verbose_logger.debug( - f"Azure AI Model Router flat cost: ${router_flat_cost:.6f} " - f"({usage.prompt_tokens} tokens × ${router_flat_cost / usage.prompt_tokens:.9f}/token)" - ) - - # Add flat cost to prompt cost - prompt_cost += router_flat_cost - - return prompt_cost, completion_cost + base_cost: Final = _base_cost_per_token(model=model, usage=usage, service_tier=service_tier) + prompt_cost, completion_cost = base_cost if base_cost is not None else (0.0, 0.0) + if not is_router_request or (base_cost is not None and _prices_router_fee_itself(model)): + return prompt_cost, completion_cost + router_flat_cost: Final = calculate_azure_model_router_flat_cost(request_model or model, usage.prompt_tokens) + return prompt_cost + router_flat_cost, completion_cost diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b1ffc1583e4..6649fa831d7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3581,6 +3581,82 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure_ai/gpt-chat-latest": { + "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "azure_ai/codex-mini": { + "cache_read_input_token_cost": 3.75e-07, + "deprecation_date": "2026-11-15", + "input_cost_per_token": 1.5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "responses", + "output_cost_per_token": 6e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure_ai/whisper": { + "deprecation_date": "2026-12-15", + "input_cost_per_second": 0.0001, + "litellm_provider": "azure_ai", + "mode": "audio_transcription", + "output_cost_per_second": 0.0001, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" + }, "azure_ai/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, @@ -3991,6 +4067,17 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, + "azure_ai/model-router": { + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 0, + "litellm_provider": "azure_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", + "comment": "Catalog-name twin of azure_ai/model_router: the flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it" + }, "azure/eu/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.375e-06, @@ -10302,6 +10389,18 @@ "/v1/ocr" ] }, + "azure_ai/cohere-command-a": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 131072, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", + "supports_function_calling": true, + "supports_tool_choice": true + }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.0015, @@ -10653,6 +10752,37 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/grok-4-20-reasoning": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 262000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_reasoning": true + }, + "azure_ai/grok-4-20-non-reasoning": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 262000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b1ffc1583e4..6649fa831d7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3581,6 +3581,82 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure_ai/gpt-chat-latest": { + "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "azure_ai/codex-mini": { + "cache_read_input_token_cost": 3.75e-07, + "deprecation_date": "2026-11-15", + "input_cost_per_token": 1.5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "responses", + "output_cost_per_token": 6e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure_ai/whisper": { + "deprecation_date": "2026-12-15", + "input_cost_per_second": 0.0001, + "litellm_provider": "azure_ai", + "mode": "audio_transcription", + "output_cost_per_second": 0.0001, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" + }, "azure_ai/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, @@ -3991,6 +4067,17 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, + "azure_ai/model-router": { + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 0, + "litellm_provider": "azure_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", + "comment": "Catalog-name twin of azure_ai/model_router: the flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it" + }, "azure/eu/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.375e-06, @@ -10302,6 +10389,18 @@ "/v1/ocr" ] }, + "azure_ai/cohere-command-a": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 131072, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", + "supports_function_calling": true, + "supports_tool_choice": true + }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.0015, @@ -10653,6 +10752,37 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/grok-4-20-reasoning": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 262000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_reasoning": true + }, + "azure_ai/grok-4-20-non-reasoning": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 262000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 9612d97d946..80cd99bd46b 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -528,3 +528,29 @@ def test_mai_thinking_1_model_info_and_cost(local_model_cost_map): assert model_info["supports_function_calling"] is True assert prompt_cost == pytest.approx(2.0) assert completion_cost == pytest.approx(8.0) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) +def test_router_entry_as_response_model_charges_the_fee_once(router_entry_name: str) -> None: + usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) + prompt_cost, completion_cost = cost_per_token(model=router_entry_name, usage=usage) + assert prompt_cost == pytest.approx(0.14, rel=1e-9) + assert completion_cost == 0.0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_unmapped_router_deployment_name_still_charges_the_fee() -> None: + usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) + prompt_cost, completion_cost = cost_per_token(model="azure-model-router", usage=usage) + assert prompt_cost == pytest.approx(0.14, rel=1e-9) + assert completion_cost == 0.0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_routed_model_response_adds_the_fee_on_top() -> None: + usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) + routed_prompt_cost, _ = cost_per_token(model="gpt-5-nano", usage=usage) + prompt_cost, _ = cost_per_token(model="gpt-5-nano", usage=usage, request_model="azure_ai/model-router") + assert routed_prompt_cost > 0 + assert prompt_cost == pytest.approx(routed_prompt_cost + 0.14, rel=1e-9) diff --git a/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py new file mode 100644 index 00000000000..9c5ca26a89c --- /dev/null +++ b/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py @@ -0,0 +1,176 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm import cost_per_token, get_model_info +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + +REPO_ROOT: Final = Path(__file__).parents[2] +COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) +AZURE_OPENAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" +FOUNDRY_AOAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/" +FOUNDRY_COHERE_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/" +FOUNDRY_GROK_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/" + + +@dataclass(frozen=True, slots=True) +class TokenPricedCatalogModel: + catalog_name: str + mode: str + source: str + input_cost_per_token: float + output_cost_per_token: float + max_input_tokens: int + max_output_tokens: int + cache_read_input_token_cost: float | None + supported_flags: tuple[str, ...] + + +TOKEN_PRICED_MODELS: Final = ( + TokenPricedCatalogModel( + catalog_name="gpt-chat-latest", + mode="chat", + source=AZURE_OPENAI_PRICING, + input_cost_per_token=5e-06, + output_cost_per_token=3e-05, + max_input_tokens=200000, + max_output_tokens=128000, + cache_read_input_token_cost=5e-07, + supported_flags=( + "supports_function_calling", + "supports_prompt_caching", + "supports_reasoning", + "supports_response_schema", + "supports_tool_choice", + "supports_vision", + "supports_web_search", + ), + ), + TokenPricedCatalogModel( + catalog_name="codex-mini", + mode="responses", + source=AZURE_OPENAI_PRICING, + input_cost_per_token=1.5e-06, + output_cost_per_token=6e-06, + max_input_tokens=200000, + max_output_tokens=100000, + cache_read_input_token_cost=3.75e-07, + supported_flags=("supports_function_calling", "supports_prompt_caching", "supports_reasoning", "supports_vision"), + ), + TokenPricedCatalogModel( + catalog_name="model-router", + mode="chat", + source=FOUNDRY_AOAI_PRICING, + input_cost_per_token=1.4e-07, + output_cost_per_token=0.0, + max_input_tokens=1048576, + max_output_tokens=32768, + cache_read_input_token_cost=None, + supported_flags=(), + ), + TokenPricedCatalogModel( + catalog_name="cohere-command-a", + mode="chat", + source=FOUNDRY_COHERE_PRICING, + input_cost_per_token=2.5e-06, + output_cost_per_token=1e-05, + max_input_tokens=131072, + max_output_tokens=4096, + cache_read_input_token_cost=None, + supported_flags=("supports_function_calling", "supports_tool_choice"), + ), + TokenPricedCatalogModel( + catalog_name="grok-4-20-reasoning", + mode="chat", + source=FOUNDRY_GROK_PRICING, + input_cost_per_token=1.25e-06, + output_cost_per_token=2.5e-06, + max_input_tokens=262000, + max_output_tokens=8192, + cache_read_input_token_cost=None, + supported_flags=( + "supports_function_calling", + "supports_reasoning", + "supports_response_schema", + "supports_tool_choice", + "supports_vision", + "supports_web_search", + ), + ), + TokenPricedCatalogModel( + catalog_name="grok-4-20-non-reasoning", + mode="chat", + source=FOUNDRY_GROK_PRICING, + input_cost_per_token=1.25e-06, + output_cost_per_token=2.5e-06, + max_input_tokens=262000, + max_output_tokens=8192, + cache_read_input_token_cost=None, + supported_flags=( + "supports_function_calling", + "supports_response_schema", + "supports_tool_choice", + "supports_vision", + "supports_web_search", + ), + ), +) +CATALOG_NAMES: Final = tuple(spec.catalog_name for spec in TOKEN_PRICED_MODELS) + ("whisper",) + + +def _cost_map_entry(path: Path, catalog_name: str) -> dict[str, object]: + return COST_MAP_ADAPTER.validate_json(path.read_bytes())[f"azure_ai/{catalog_name}"] + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) +def test_azure_ai_catalog_name_is_priced_and_routed(spec: TokenPricedCatalogModel) -> None: + routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{spec.catalog_name}") + assert (routed_model, provider) == (spec.catalog_name, "azure_ai") + + info = get_model_info(model=routed_model, custom_llm_provider=provider) + assert info["litellm_provider"] == "azure_ai" + assert info["mode"] == spec.mode + assert info["input_cost_per_token"] == spec.input_cost_per_token + assert info["output_cost_per_token"] == spec.output_cost_per_token + assert info["cache_read_input_token_cost"] == spec.cache_read_input_token_cost + assert info["max_input_tokens"] == spec.max_input_tokens + assert info["max_output_tokens"] == spec.max_output_tokens + assert info["max_tokens"] == spec.max_output_tokens + for flag in spec.supported_flags: + assert info[flag] is True, flag + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + "spec", [spec for spec in TOKEN_PRICED_MODELS if spec.catalog_name != "model-router"], ids=lambda spec: spec.catalog_name +) +def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: + prompt_cost, completion_cost = cost_per_token( + model=f"azure_ai/{spec.catalog_name}", prompt_tokens=1_000_000, completion_tokens=1_000_000 + ) + assert prompt_cost == pytest.approx(spec.input_cost_per_token * 1_000_000) + assert completion_cost == pytest.approx(spec.output_cost_per_token * 1_000_000) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None: + routed_model, provider, _, _ = get_llm_provider(model="azure_ai/whisper") + assert (routed_model, provider) == ("whisper", "azure_ai") + + info = get_model_info(model=routed_model, custom_llm_provider=provider) + assert info["mode"] == "audio_transcription" + assert info["input_cost_per_second"] == 0.0001 + assert info["output_cost_per_second"] == 0.0001 + + +@pytest.mark.parametrize("catalog_name", CATALOG_NAMES) +def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> None: + main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", catalog_name) + backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json", catalog_name) + + assert str(main_entry["source"]).startswith("https://azure.microsoft.com/en-us/pricing/details/") + assert backup_entry == main_entry From 3829ebdce54a0330ca2c9d60a5bbee94c2b09382 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:26:23 -0700 Subject: [PATCH 075/411] fix(token_counter): estimate over-cap strings from evenly spaced samples instead of the prefix A string above TOKEN_COUNTER_MAX_EXACT_CHARS was counted from its first cap characters and scaled, so a string whose start tokenizes unlike its end got a skewed count, and that count reaches fallback billing when the provider sends no usage. The estimate now tokenizes 16 evenly spaced samples that together total the cap and scales their sum by the string's length, keeping the same bound on work while tracking the whole string --- litellm/litellm_core_utils/token_counter.py | 16 ++++++++- .../litellm_core_utils/test_token_counter.py | 36 +++++++++++-------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index d526f0f2996..e3a0d4c785a 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -315,6 +315,8 @@ def calculate_img_tokens( TokenCounterFunction = Callable[[str], int] + +EXTRAPOLATION_SAMPLES: Final = 16 """ Type for a function that counts tokens in a string. """ @@ -547,11 +549,23 @@ def _get_extrapolating_count_function( def count_tokens(text: str) -> int: if len(text) <= max_exact_chars: return count_exactly(text) - return round(count_exactly(text[:max_exact_chars]) * len(text) / max_exact_chars) + samples: Final = _evenly_spaced_samples(text, max_exact_chars) + sampled_chars: Final = sum(len(sample) for sample in samples) + return round(sum(count_exactly(sample) for sample in samples) * len(text) / sampled_chars) return count_tokens +def _evenly_spaced_samples(text: str, total_chars: int) -> tuple[str, ...]: + sample_count: Final = min(EXTRAPOLATION_SAMPLES, total_chars) + sample_chars: Final = total_chars // sample_count + last_start: Final = len(text) - sample_chars + return tuple( + text[start : start + sample_chars] + for start in (last_start * index // max(sample_count - 1, 1) for index in range(sample_count)) + ) + + def _get_count_function( model: str | None, custom_tokenizer: dict | SelectTokenizerResponse | None = None, diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index fddeacc37f9..5da8413ac9b 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -153,27 +153,35 @@ async def test_huggingface_count_in_a_worker_thread_leaves_the_event_loop_free() assert max(lags) < took / 4, f"the event loop stalled {max(lags):.3f}s during a {took:.3f}s count" -@pytest.mark.parametrize( - ("max_exact_chars", "expected"), - [(1_000, 10_000), (5_000, 6_000), (10_000, 6_000)], -) -def test_count_above_the_cap_scales_the_exact_count_of_the_prefix(max_exact_chars, expected): - def count_exactly(chunk: str) -> int: - return chunk.count("a") + len(chunk) +@pytest.mark.parametrize("max_exact_chars", [64, 1_000, 2_500]) +def test_count_above_the_cap_samples_the_whole_string_and_scales(max_exact_chars): + count_exactly: Final = MagicMock(side_effect=lambda chunk: chunk.count("a") + len(chunk)) + front_heavy: Final = "a" * 1_000 + "b" * 4_000 + exact: Final = 1_000 + len(front_heavy) - count_tokens: Final = _get_extrapolating_count_function(count_exactly, max_exact_chars=max_exact_chars) + estimate: Final = _get_extrapolating_count_function(count_exactly, max_exact_chars=max_exact_chars)(front_heavy) - assert count_tokens("a" * 1_000 + "b" * 4_000) == expected + assert abs(estimate - exact) <= exact // 100 + assert sum(len(call.args[0]) for call in count_exactly.call_args_list) <= max_exact_chars + + +def test_count_at_or_below_the_cap_is_exact(): + count_exactly: Final = MagicMock(side_effect=len) + + assert _get_extrapolating_count_function(count_exactly, max_exact_chars=5_000)("a" * 5_000) == 5_000 + assert count_exactly.call_args_list == [(("a" * 5_000,),)] def test_token_counter_applies_the_default_cap(): max_exact_chars: Final = litellm.constants.TOKEN_COUNTER_MAX_EXACT_CHARS - prefix: Final = ("The quick brown fox jumps over the lazy dog. " * (max_exact_chars // 45 + 1))[:max_exact_chars] - over_the_cap: Final = prefix + "a" * 200_000 - scaled: Final = round(token_counter_new(model="gpt-5.6", text=prefix) * len(over_the_cap) / max_exact_chars) + prose: Final = ("The quick brown fox jumps over the lazy dog. " * (max_exact_chars // 45 + 1))[:max_exact_chars] + over_the_cap: Final = prose + "a" * 200_000 + exact: Final = _get_exact_count_function("gpt-5.6")(over_the_cap) - assert token_counter_new(model="gpt-5.6", text=over_the_cap) == scaled - assert _get_exact_count_function("gpt-5.6")(over_the_cap) != scaled + estimate: Final = token_counter_new(model="gpt-5.6", text=over_the_cap) + + assert estimate != exact + assert abs(estimate - exact) <= exact // 100 @pytest.mark.parametrize( From 0d89873daf9e82a692825820adca2a47c0aab8dc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:36:09 -0700 Subject: [PATCH 076/411] test(token_counter): type the parametrized cap test arguments --- tests/test_litellm/litellm_core_utils/test_token_counter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 5da8413ac9b..db87bc52600 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -154,7 +154,7 @@ async def test_huggingface_count_in_a_worker_thread_leaves_the_event_loop_free() @pytest.mark.parametrize("max_exact_chars", [64, 1_000, 2_500]) -def test_count_above_the_cap_samples_the_whole_string_and_scales(max_exact_chars): +def test_count_above_the_cap_samples_the_whole_string_and_scales(max_exact_chars: int): count_exactly: Final = MagicMock(side_effect=lambda chunk: chunk.count("a") + len(chunk)) front_heavy: Final = "a" * 1_000 + "b" * 4_000 exact: Final = 1_000 + len(front_heavy) @@ -188,7 +188,7 @@ def test_token_counter_applies_the_default_cap(): ("configured", "expected"), [("2048", 2048), ("0", 4_000_000), ("not-an-int", 4_000_000)], ) -def test_max_exact_chars_config_is_honoured(monkeypatch, configured, expected): +def test_max_exact_chars_config_is_honoured(monkeypatch: pytest.MonkeyPatch, configured: str, expected: int): monkeypatch.setenv("TOKEN_COUNTER_MAX_EXACT_CHARS", configured) try: assert importlib.reload(litellm.constants).TOKEN_COUNTER_MAX_EXACT_CHARS == expected From 95402ccb711cdcd93f1c296579b18ec8bd32cab4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:36:40 -0700 Subject: [PATCH 077/411] test(azure_ai): move the Foundry catalog metadata test into the mapped azure_ai directory The new metadata test sat at the top of tests/test_litellm. The azure_ai metadata tests live in tests/test_litellm/llms/azure_ai next to the cost calculator test, so this moves it there and bumps its repo-root lookup by the two extra directory levels. No test changes. --- .../azure_ai}/test_azure_ai_foundry_catalog_model_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/test_litellm/{ => llms/azure_ai}/test_azure_ai_foundry_catalog_model_metadata.py (99%) diff --git a/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py similarity index 99% rename from tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py rename to tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 9c5ca26a89c..1b4e83438a6 100644 --- a/tests/test_litellm/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -8,7 +8,7 @@ from pydantic import TypeAdapter from litellm import cost_per_token, get_model_info from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -REPO_ROOT: Final = Path(__file__).parents[2] +REPO_ROOT: Final = Path(__file__).parents[4] COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) AZURE_OPENAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" FOUNDRY_AOAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/" From 12d9413860e96f1cf2b0d138466b34fcae53f925 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:37:58 -0700 Subject: [PATCH 078/411] refactor(passthrough): share the relay shape table and price FLUX 2 provider relays --- .../llms/azure/passthrough/transformation.py | 44 +++----------- .../azure_ai/passthrough/transformation.py | 36 ++++-------- .../base_llm/passthrough/transformation.py | 58 ++++++++++++++----- ...est_azure_ai_passthrough_transformation.py | 49 +++++++++++++--- 4 files changed, 106 insertions(+), 81 deletions(-) diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index d7078d94a0b..bdb854bb87a 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -1,5 +1,4 @@ -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final, Optional import httpx @@ -10,7 +9,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.base_llm.passthrough.transformation import ( BasePassthroughConfig, - relayed_json_object, + RelayShape, + logged_relay_shape, replace_path_segment, strip_leading_model_segment, ) @@ -22,6 +22,7 @@ from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse if TYPE_CHECKING: from httpx import URL + from litellm.llms.base_llm.passthrough.transformation import LoggedRelayResponse from litellm.types.utils import CostResponseTypes @@ -41,40 +42,13 @@ def _relayed_messages(litellm_logging_obj: Logging) -> Sequence[Mapping[str, obj return details.request_data.messages if details.request_data else None -@dataclass(frozen=True, slots=True) -class OpenAIRelayShape: - path_suffix: str - call_type: CallTypes - parse: Callable[[Mapping[str, object]], EmbeddingResponse | ImageResponse | ResponsesAPIResponse] - - OPENAI_RELAY_SHAPES: Final = ( - OpenAIRelayShape("/embeddings", CallTypes.aembedding, EmbeddingResponse.model_validate), - OpenAIRelayShape("/responses", CallTypes.aresponses, ResponsesAPIResponse.model_validate), - OpenAIRelayShape("/images/generations", CallTypes.aimage_generation, ImageResponse.model_validate), + RelayShape("/embeddings", CallTypes.aembedding, EmbeddingResponse.model_validate), + RelayShape("/responses", CallTypes.aresponses, ResponsesAPIResponse.model_validate), + RelayShape("/images/generations", CallTypes.aimage_generation, ImageResponse.model_validate), ) -def logged_openai_response( - httpx_response: Response, logging_obj: Logging, endpoint: str -) -> EmbeddingResponse | ImageResponse | ResponsesAPIResponse | None: - relayed_path: Final = f"/{endpoint.strip('/')}" - shape: Final = next( - (candidate for candidate in OPENAI_RELAY_SHAPES if relayed_path.endswith(candidate.path_suffix)), None - ) - body: Final = relayed_json_object(httpx_response) if shape else None - if shape is None or body is None: - return None - try: - parsed: Final = shape.parse(body) - except ValidationError: - return None - logging_obj.call_type = ( - shape.call_type.value - ) # rebind-ok: routes cost calculation to the relayed shape's pricing path - return parsed - - class AzurePassthroughConfig(BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: return bool(request_data.get("stream")) @@ -151,13 +125,13 @@ class AzurePassthroughConfig(BasePassthroughConfig): request_data: dict, logging_obj: Logging, endpoint: str, - ) -> Optional["CostResponseTypes | ResponsesAPIResponse"]: + ) -> Optional["LoggedRelayResponse"]: from litellm import encoding from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.utils import ModelResponse if "chat/completions" not in endpoint: - return logged_openai_response(httpx_response, logging_obj, endpoint) + return logged_relay_shape(OPENAI_RELAY_SHAPES, httpx_response, logging_obj, endpoint) openai_chat_config: Final = OpenAIGPTConfig() diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index c49e10566ed..e854874eaee 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -16,19 +16,20 @@ from litellm.llms.azure_ai.common_utils import ( from litellm.llms.azure_ai.ocr.common_utils import get_azure_ai_ocr_config from litellm.llms.base_llm.passthrough.transformation import ( BasePassthroughConfig, - relayed_json_object, + RelayShape, + logged_relay_shape, strip_leading_model_segment, ) from litellm.types.llms.openai import AllMessageValues from litellm.types.rerank import RerankResponse -from litellm.types.utils import CallTypes, StandardPassThroughResponseObject +from litellm.types.utils import CallTypes, ImageResponse, StandardPassThroughResponseObject if TYPE_CHECKING: from httpx import URL, Response from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse - from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.llms.base_llm.passthrough.transformation import LoggedRelayResponse from litellm.types.utils import CostResponseTypes @@ -83,16 +84,10 @@ def relayed_body(httpx_response: Response) -> str | dict: return body if isinstance(body, dict) else httpx_response.text -def logged_rerank_response(httpx_response: Response, logging_obj: Logging, endpoint: str) -> RerankResponse | None: - body: Final = relayed_json_object(httpx_response) if f"/{endpoint.strip('/')}".endswith("/rerank") else None - if body is None: - return None - try: - rerank_response: Final = RerankResponse.model_validate(body) - except ValidationError: - return None - logging_obj.call_type = CallTypes.arerank.value # rebind-ok: routes cost calculation to the per-query rerank path - return rerank_response +FOUNDRY_RELAY_SHAPES: Final = ( + RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate), + RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate), +) class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): @@ -148,14 +143,7 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): request_data: Mapping[str, object], logging_obj: Logging, endpoint: str, - ) -> ( - CostResponseTypes - | OCRResponse - | RerankResponse - | ResponsesAPIResponse - | StandardPassThroughResponseObject - | None - ): + ) -> LoggedRelayResponse | OCRResponse | StandardPassThroughResponseObject | None: from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig chat_result: Final = AzurePassthroughConfig().logging_non_streaming_response( # pyright: ignore[reportUnknownMemberType] # the Azure config still types request_data as a bare dict @@ -171,9 +159,9 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): ocr_result: Final = self.logged_ocr_response(model, httpx_response, logging_obj, endpoint) if ocr_result is not None: return ocr_result - rerank_result: Final = logged_rerank_response(httpx_response, logging_obj, endpoint) - if rerank_result is not None: - return rerank_result + foundry_result: Final = logged_relay_shape(FOUNDRY_RELAY_SHAPES, httpx_response, logging_obj, endpoint) + if foundry_result is not None: + return foundry_result return StandardPassThroughResponseObject(response=relayed_body(httpx_response)) def logged_ocr_response( diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index 975b74137db..93ec5a09e62 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -1,10 +1,15 @@ +from __future__ import annotations + import re from abc import abstractmethod -from collections.abc import Mapping -from typing import TYPE_CHECKING, Final, Optional, Union +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final, TypeAlias from pydantic import TypeAdapter, ValidationError +from litellm.types.utils import CallTypes + from ..base_utils import BaseLLMModelInfo if TYPE_CHECKING: @@ -18,6 +23,8 @@ if TYPE_CHECKING: from ..chat.transformation import BaseLLMException from ..ocr.transformation import OCRResponse + LoggedRelayResponse: TypeAlias = CostResponseTypes | RerankResponse | ResponsesAPIResponse + RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) @@ -39,7 +46,7 @@ def replace_path_segment(endpoint: str, segment: str, replacement: str) -> str: return bounded_segment.sub(lambda _: replacement, endpoint) -def relayed_json_object(httpx_response: "Response") -> Mapping[str, object] | None: +def relayed_json_object(httpx_response: Response) -> Mapping[str, object] | None: if httpx_response.status_code != 200: return None try: @@ -48,6 +55,31 @@ def relayed_json_object(httpx_response: "Response") -> Mapping[str, object] | No return None +@dataclass(frozen=True, slots=True) +class RelayShape: + path_suffix: str + call_type: CallTypes + parse: Callable[[Mapping[str, object]], LoggedRelayResponse] + + +def logged_relay_shape( + shapes: Sequence[RelayShape], httpx_response: Response, logging_obj: LiteLLMLoggingObj, endpoint: str +) -> LoggedRelayResponse | None: + relayed_path: Final = f"/{endpoint.strip('/')}" + shape: Final = next((candidate for candidate in shapes if relayed_path.endswith(candidate.path_suffix)), None) + body: Final = relayed_json_object(httpx_response) if shape else None + if shape is None or body is None: + return None + try: + parsed: Final = shape.parse(body) + except ValidationError: + return None + logging_obj.call_type = ( + shape.call_type.value + ) # rebind-ok: routes cost calculation to the relayed shape's pricing path + return parsed + + class BasePassthroughConfig(BaseLLMModelInfo): @abstractmethod def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: @@ -60,7 +92,7 @@ class BasePassthroughConfig(BaseLLMModelInfo): endpoint: str, base_target_url: str, request_query_params: Mapping[str, object] | None, - ) -> "URL": + ) -> URL: """ Helper function to add query params to the url Args: @@ -94,7 +126,7 @@ class BasePassthroughConfig(BaseLLMModelInfo): endpoint: str, request_query_params: dict | None, litellm_params: dict, - ) -> tuple["URL", str]: + ) -> tuple[URL, str]: """ Get the complete url for the request Returns: @@ -124,9 +156,7 @@ class BasePassthroughConfig(BaseLLMModelInfo): """ return headers, None - def get_error_class( - self, error_message: str, status_code: int, headers: Union[dict, "Headers"] - ) -> "BaseLLMException": + def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException: from litellm.llms.base_llm.chat.transformation import BaseLLMException return BaseLLMException(status_code=status_code, message=error_message, headers=headers) @@ -135,23 +165,21 @@ class BasePassthroughConfig(BaseLLMModelInfo): self, model: str, custom_llm_provider: str, - httpx_response: "Response", + httpx_response: Response, request_data: dict, - logging_obj: "LiteLLMLoggingObj", + logging_obj: LiteLLMLoggingObj, endpoint: str, - ) -> Optional[ - "CostResponseTypes | OCRResponse | RerankResponse | ResponsesAPIResponse | StandardPassThroughResponseObject" - ]: + ) -> LoggedRelayResponse | OCRResponse | StandardPassThroughResponseObject | None: pass def handle_logging_collected_chunks( self, all_chunks: list[str], - litellm_logging_obj: "LiteLLMLoggingObj", + litellm_logging_obj: LiteLLMLoggingObj, model: str, custom_llm_provider: str, endpoint: str, - ) -> Optional["CostResponseTypes"]: + ) -> CostResponseTypes | None: return None def _convert_raw_bytes_to_str_lines(self, raw_bytes: list[bytes]) -> list[str]: diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index 788aced50f8..8b6c76952d6 100644 --- a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -25,7 +25,9 @@ def clear_azure_ai_env(monkeypatch): def test_provider_config_manager_resolves_azure_ai_passthrough_config(): - config = ProviderConfigManager.get_provider_passthrough_config(model="Cohere-parse-v5", provider=LlmProviders.AZURE_AI) + config = ProviderConfigManager.get_provider_passthrough_config( + model="Cohere-parse-v5", provider=LlmProviders.AZURE_AI + ) assert isinstance(config, AzureAIPassthroughConfig) @@ -189,7 +191,10 @@ def test_no_credentials_at_all_raises(): [({"stream": True}, True), ({"stream": 1}, True), ({"stream": False}, False), ({}, False)], ) def test_is_streaming_request_reads_the_stream_flag(request_data, expected): - assert AzureAIPassthroughConfig().is_streaming_request(endpoint="models/chat/completions", request_data=request_data) is expected + assert ( + AzureAIPassthroughConfig().is_streaming_request(endpoint="models/chat/completions", request_data=request_data) + is expected + ) def _chat_completion_response() -> httpx.Response: @@ -266,7 +271,12 @@ def _relay_logging_obj(model: str, api_base: str) -> Logging: def _relay_logging_result( - config: AzureAIPassthroughConfig, model: str, native_path: str, body, api_base: str = FOUNDRY_BASE, status_code: int = 200 + config: AzureAIPassthroughConfig, + model: str, + native_path: str, + body, + api_base: str = FOUNDRY_BASE, + status_code: int = 200, ): relayed_url = f"{FOUNDRY_BASE}/{native_path}?api-version=2024-05-01-preview" logging_obj = _relay_logging_obj(model, api_base) @@ -355,7 +365,11 @@ def test_deployment_without_an_ocr_config_is_never_costed_as_ocr(): def test_accepted_ocr_job_without_a_result_body_is_not_costed(): result, logging_obj = _relay_logging_result( - AzureAIPassthroughConfig(), "mistral-document-ai-2512", "providers/mistral/azure/ocr", {"status": "running"}, status_code=202 + AzureAIPassthroughConfig(), + "mistral-document-ai-2512", + "providers/mistral/azure/ocr", + {"status": "running"}, + status_code=202, ) assert result == {"response": {"status": "running"}} @@ -364,7 +378,10 @@ def test_accepted_ocr_job_without_a_result_body_is_not_costed(): def test_unparseable_ocr_body_falls_back_to_the_passthrough_object(): result, logging_obj = _relay_logging_result( - AzureAIPassthroughConfig(), "mistral-document-ai-2512", "providers/mistral/azure/ocr", ["not", "an", "ocr", "body"] + AzureAIPassthroughConfig(), + "mistral-document-ai-2512", + "providers/mistral/azure/ocr", + ["not", "an", "ocr", "body"], ) assert result == {"response": '["not", "an", "ocr", "body"]'} @@ -423,6 +440,17 @@ def test_image_generation_relay_is_costed_per_image(): assert logging_obj._response_cost_calculator(result=result) == pytest.approx(per_image) +def test_flux_2_relay_through_the_provider_route_is_costed_per_image(): + result, logging_obj = _relay_logging_result( + AzureAIPassthroughConfig(), "FLUX.2-pro", "providers/blackforestlabs/v1/flux-2-pro", IMAGE_BODY + ) + per_image = litellm.get_model_info("azure_ai/FLUX.2-pro")["output_cost_per_image"] + + assert isinstance(result, ImageResponse) + assert logging_obj.call_type == "aimage_generation" + assert logging_obj._response_cost_calculator(result=result) == pytest.approx(per_image) + + def test_rejected_rerank_relay_keeps_the_passthrough_object_and_call_type(): result, logging_obj = _relay_logging_result( AzureAIPassthroughConfig(), @@ -439,8 +467,15 @@ def test_rejected_rerank_relay_keeps_the_passthrough_object_and_call_type(): def test_streaming_chat_completion_chunks_are_costed_like_azure(): head = {"id": "chatcmpl-1", "object": "chat.completion.chunk", "created": 1, "model": "gpt-5.4-mini"} chunks = [ - "data: " + json.dumps({**head, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}]}), - "data: " + json.dumps({**head, "choices": [], "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}}), + "data: " + + json.dumps( + { + **head, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + } + ), + "data: " + + json.dumps({**head, "choices": [], "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}}), "data: [DONE]", ] From ba91588b15d308daf229f464b9aa89a0f7480afd Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 7 Sep 2026 18:38:12 -0700 Subject: [PATCH 079/411] fix(proxy): price one cost estimate at one moment The totals, the per-token-type lines and the reported rates each resolved off-peak pricing on their own clock read, so a quote taken as a window opened could bill on one side of the boundary and report rates from the other. /cost/estimate now pins a billing moment for the whole quote, and every rate lookup answers for the pinned moment instead of reading the clock again Claude-Session: https://claude.ai/code/session_011Tn3657NkV6ojLqewL64Kb --- litellm/_internal_context.py | 24 +++++++++++ .../litellm_core_utils/llm_cost_calc/utils.py | 9 ++-- .../cost_tracking_settings.py | 43 +++++++++++-------- .../llm_cost_calc/test_llm_cost_calc_utils.py | 42 ++++++++++++++++++ .../test_cost_tracking_settings.py | 30 +++++++++++++ 5 files changed, 126 insertions(+), 22 deletions(-) diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index f856fe0f2b3..8132008731f 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -6,9 +6,33 @@ be settable from user input. Context variables are scoped to the current asyncio task and cannot be injected via HTTP request bodies. """ +from collections.abc import Generator +from contextlib import contextmanager from contextvars import ContextVar +from datetime import datetime, timezone from typing import Final # When True, suppresses async logging and billing for internal sub-calls # (e.g., emulated file-search steps that make nested LLM calls). is_internal_call: Final[ContextVar[bool]] = ContextVar("is_internal_call", default=False) + +# One request prices its totals, its per-token-type lines and the rates it reports on +# separate code paths. Each reads the clock for off-peak pricing, so without a pinned +# moment they can land on either side of a window boundary and disagree with each other. +_billing_time: Final[ContextVar[datetime | None]] = ContextVar("billing_time", default=None) + + +@contextmanager +def pinned_billing_time(moment: datetime) -> Generator[None]: + """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read.""" + token: Final = _billing_time.set(moment) + try: + yield + finally: + _billing_time.reset(token) + + +def current_billing_time() -> datetime: + """The pinned billing moment, or now in UTC outside a pinned block.""" + pinned: Final = _billing_time.get() + return pinned if pinned is not None else datetime.now(timezone.utc) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 2bb15fb4c48..e2168528e6b 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -10,6 +10,7 @@ from typing import Any, Final, Literal, TypedDict, cast from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import litellm +from litellm._internal_context import current_billing_time from litellm._logging import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import ( select_tier_for_input, @@ -306,7 +307,7 @@ def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_ than being localised, so callers must pass datetime.now(timezone.utc), never datetime.now(), or every window shifts by the host's offset. """ - reference: Final = current_time if current_time is not None else datetime.now(timezone.utc) + reference: Final = current_time if current_time is not None else current_billing_time() now: Final = (reference.astimezone(timezone.utc) if reference.tzinfo is not None else reference).time() windows: Final = (off_peak_hours_utc,) if isinstance(off_peak_hours_utc, str) else off_peak_hours_utc for window in windows: @@ -393,7 +394,7 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None = rules: the flat hours_utc windows, which apply every day, or any entry in windows, whose hours apply only on its weekdays. """ - reference: Final = current_time if current_time is not None else datetime.now(timezone.utc) + reference: Final = current_time if current_time is not None else current_billing_time() reference_utc: Final = ( reference.astimezone(timezone.utc) if reference.tzinfo is not None else reference.replace(tzinfo=timezone.utc) ) @@ -1187,7 +1188,7 @@ def generic_cost_per_token( usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0 ) - billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc) + billing_time: Final = current_time if current_time is not None else current_billing_time() ( prompt_base_cost, completion_base_cost, @@ -1379,7 +1380,7 @@ def _cost_map_billed_rates( vertex_location: str | None, current_time: datetime | None, ) -> BilledTokenRates: - billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc) + billing_time: Final = current_time if current_time is not None else current_billing_time() ( prompt_base_cost, completion_base_cost, diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 17d82fd17e3..1faa66584d5 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -18,6 +18,7 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel import litellm +from litellm._internal_context import current_billing_time, pinned_billing_time from litellm._logging import verbose_proxy_logger from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.llm_cost_calc.utils import get_billed_token_rates @@ -631,33 +632,39 @@ async def estimate_cost( function_id="cost-estimate", ) - # Use completion_cost which handles all the logic including margins/discounts - try: - cost_per_request: Final = completion_cost( - completion_response=mock_response, + # The totals, the per-token-type lines and the reported rates each resolve pricing on their + # own path. Pinning one moment keeps an off-peak window that opens mid-quote from splitting them. + billed_at: Final = current_billing_time() + with pinned_billing_time(billed_at): + # Use completion_cost which handles all the logic including margins/discounts + try: + cost_per_request: Final = completion_cost( + completion_response=mock_response, + model=resolved_model, + custom_llm_provider=resolved_provider, + custom_cost_per_token=resolved.custom_cost_per_token, + litellm_logging_obj=litellm_logging_obj, + ) + except Exception as e: + raise HTTPException( + status_code=404, + detail={ + "error": f"Could not calculate cost for model '{request.model}' (resolved to '{resolved_model}'): {e}" + }, + ) + + rates: Final = get_billed_token_rates( model=resolved_model, custom_llm_provider=resolved_provider, + usage=usage, custom_cost_per_token=resolved.custom_cost_per_token, - litellm_logging_obj=litellm_logging_obj, - ) - except Exception as e: - raise HTTPException( - status_code=404, - detail={ - "error": f"Could not calculate cost for model '{request.model}' (resolved to '{resolved_model}'): {e}" - }, + current_time=billed_at, ) per_request: Final = _cost_lines(cost_per_request, litellm_logging_obj.cost_breakdown) daily: Final = per_request.times(request.num_requests_per_day) monthly: Final = per_request.times(request.num_requests_per_month) - rates: Final = get_billed_token_rates( - model=resolved_model, - custom_llm_provider=resolved_provider, - usage=usage, - custom_cost_per_token=resolved.custom_cost_per_token, - ) model_info: Final = _lookup_model_info(resolved_model) mapped_provider: Final = model_info.get("litellm_provider") if model_info is not None else None custom_llm_provider: Final = mapped_provider if mapped_provider is not None else resolved_provider diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 4de3c059e63..90178428018 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,9 +1,11 @@ import json +from datetime import datetime, timezone import pytest from fastapi.testclient import TestClient import litellm +from litellm._internal_context import pinned_billing_time from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -3981,6 +3983,46 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp assert breakdown.reasoning_cost == pytest.approx(200 * rates.output_cost_per_reasoning_token) +def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_moment(monkeypatch): + """Totals and reported rates resolve off-peak pricing on separate paths that each read the + clock, so a window opening between the two reads used to leave them describing one request + at two different prices. Pinned, both must answer for the pinned moment.""" + monkeypatch.setitem( + litellm.model_cost, + "off-peak-model", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "off_peak_pricing": { + "hours_utc": "02:00-03:00", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 5e-6, + }, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)): + off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token( + model="off-peak-model", usage=usage, custom_llm_provider="openai" + ) + off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) + with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)): + peak_prompt_cost, peak_completion_cost = generic_cost_per_token( + model="off-peak-model", usage=usage, custom_llm_provider="openai" + ) + peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) + + assert off_peak_rates.input_cost_per_token == pytest.approx(1e-6) + assert peak_rates.input_cost_per_token == pytest.approx(3e-6) + assert off_peak_prompt_cost == pytest.approx(1000 * off_peak_rates.input_cost_per_token) + assert off_peak_completion_cost == pytest.approx(500 * off_peak_rates.output_cost_per_token) + assert peak_prompt_cost == pytest.approx(1000 * peak_rates.input_cost_per_token) + assert peak_completion_cost == pytest.approx(500 * peak_rates.output_cost_per_token) + + def test_billed_token_rates_are_none_for_an_unpriced_model(): usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index 9847c4092df..e9485f3a044 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -4,6 +4,7 @@ Tests for cost tracking settings management endpoints. Tests the GET and PATCH endpoints for managing cost discount configuration. """ +from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -12,6 +13,7 @@ from pydantic import ValidationError import litellm +from litellm._internal_context import pinned_billing_time from litellm.proxy._types import CostEstimateRequest from litellm.proxy.management_endpoints.cost_tracking_settings import router from litellm.proxy.proxy_server import app @@ -1108,6 +1110,34 @@ class TestEstimateCostCacheAndReasoningTokens: ) assert response.output_cost_per_request == pytest.approx(1_000 * response.output_cost_per_token) + @pytest.mark.asyncio + async def test_a_quote_prices_its_totals_and_its_rates_at_the_same_moment(self, monkeypatch): + """The totals and the reported rates resolve off-peak pricing on separate paths. A quote + taken as a window opens must not bill on one side of it and report rates from the other.""" + monkeypatch.setitem( + litellm.model_cost, + A_MAPPED_MODEL, + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "off_peak_pricing": { + "hours_utc": "02:00-03:00", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 5e-6, + }, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + + with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)): + response = await _estimate(None, model=A_MAPPED_MODEL) + + assert response.input_cost_per_token == pytest.approx(1e-6) + assert response.output_cost_per_token == pytest.approx(5e-6) + assert response.input_cost_per_request == pytest.approx(INPUT_TOKENS * response.input_cost_per_token) + assert response.output_cost_per_request == pytest.approx(OUTPUT_TOKENS * response.output_cost_per_token) + class TestCostEstimateRequestTokenSubsets: def test_cache_tokens_beyond_the_input_tokens_are_rejected(self): From 02b18b4fc638dfba8fd77e70db796b3ea38c8757 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:10:06 -0700 Subject: [PATCH 080/411] fix(passthrough): match image URL schemes case-insensitively when counting relayed prompt tokens --- .../openai_passthrough_logging_handler.py | 4 +++- .../test_openai_passthrough_logging_handler.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 206002d13f7..93fe3c5b31b 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -105,7 +105,9 @@ def _is_remote_high_detail_image(part: object) -> bool: if not isinstance(image_url, Mapping): return False url: Final = image_url.get("url") - return isinstance(url, str) and url.startswith(("http://", "https://")) and image_url.get("detail") == "high" + return ( + isinstance(url, str) and url.lower().startswith(("http://", "https://")) and image_url.get("detail") == "high" + ) def _content_parts(message: Mapping[str, object]) -> Sequence[object]: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index 0ddafd20e68..a3d3ae32169 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -2083,3 +2083,12 @@ def test_count_relayed_prompt_tokens_charges_only_the_remote_high_detail_image_a assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) == ( litellm.token_counter(model="gpt-4.1-mini", messages=TEXT_ONLY_MESSAGES) + high_detail_image_token_upper_bound() ) + + +@pytest.mark.parametrize("scheme", ["HTTPS://", "Http://"]) +def test_count_relayed_prompt_tokens_charges_an_uppercase_scheme_remote_high_detail_image_at_the_upper_bound(scheme): + messages = _image_messages(scheme + UNREACHABLE_IMAGE_URL.split("://", 1)[1], "high") + + assert count_relayed_prompt_tokens("gpt-4.1-mini", messages) == ( + litellm.token_counter(model="gpt-4.1-mini", messages=TEXT_ONLY_MESSAGES) + high_detail_image_token_upper_bound() + ) From dcd38ab9f0235c18fc429eeb5802519fde26c264 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:15:36 -0700 Subject: [PATCH 081/411] fix(proxy): count failure and rate-limit input tokens off the event loop The failure hook's usage estimate and the project ITPM reservation both called litellm.token_counter inline on the event loop, so a large request that failed or hit the limiter stalled the gateway the same way the count_tokens endpoints did. Both now run through asyncify. The loop-lag probe the existing tests used moves into a shared helper that warms the tokenizer first, and two new tests fail when either count runs inline --- .../hooks/parallel_request_limiter_v3.py | 3 +- litellm/proxy/utils.py | 3 +- .../litellm_core_utils/event_loop_lag.py | 39 +++++++++++++++++++ .../litellm_core_utils/test_token_counter.py | 30 +++++--------- .../proxy/hooks/test_tpm_concurrent.py | 37 ++++++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 34 ++++++---------- tests/test_litellm/proxy/test_proxy_utils.py | 38 ++++++++++++++++++ 7 files changed, 139 insertions(+), 45 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/event_loop_lag.py diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 31437af7770..74266801e50 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -28,6 +28,7 @@ from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) @@ -3303,7 +3304,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit=min_configured_otpm_limit, call_type=call_type, ) - raw_estimated_input_tokens: Final = self._estimate_precise_input_tokens( + raw_estimated_input_tokens: Final = await asyncify(self._estimate_precise_input_tokens)( data=data, model=requested_model, call_type=call_type ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ddf31cb1d8a..1014c2b9b6a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -91,6 +91,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert +from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import ( coerce_token_limit, independent_snapshot, @@ -2569,7 +2570,7 @@ class ProxyLogging: original_exception=original_exception, ) - request_data.update(_failure_fields_to_lift(request_data)) + request_data.update(await asyncify(_failure_fields_to_lift)(request_data)) # Remove before callbacks iterate — not serialisable request_data.pop("litellm_logging_obj", None) diff --git a/tests/test_litellm/litellm_core_utils/event_loop_lag.py b/tests/test_litellm/litellm_core_utils/event_loop_lag.py new file mode 100644 index 00000000000..c2bbe2204c7 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/event_loop_lag.py @@ -0,0 +1,39 @@ +import asyncio +import time +from collections.abc import Awaitable, Callable +from typing import Final, TypeVar + +import litellm + +T = TypeVar("T") + + +def warm_tokenizer(model: str) -> None: + litellm.token_counter(model=model, text="load the tokenizer before anything is timed") + + +async def loop_wake_lags(until: asyncio.Event) -> tuple[float, ...]: + async def wake_lag() -> float: + started: Final = time.perf_counter() + await asyncio.sleep(0.001) + return time.perf_counter() - started - 0.001 + + return tuple([await wake_lag() for _ in iter(until.is_set, True)]) + + +async def timed_with_loop_lags(run: Callable[[], Awaitable[T]]) -> tuple[T, float, tuple[float, ...]]: + finished: Final = asyncio.Event() + + async def timed() -> tuple[T, float]: + started: Final = time.perf_counter() + try: + return await run(), time.perf_counter() - started + finally: + finished.set() + + (result, took), lags = await asyncio.gather(timed(), loop_wake_lags(finished)) + return result, took, lags + + +def assert_loop_stayed_free(took: float, lags: tuple[float, ...]) -> None: + assert max(lags) < took / 4, f"the event loop stalled {max(lags):.3f}s during a {took:.3f}s count" diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index db87bc52600..e3dd5feb18d 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -24,6 +24,11 @@ from litellm.litellm_core_utils.token_counter import ( ) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new from tests.large_text import text +from tests.test_litellm.litellm_core_utils.event_loop_lag import ( + assert_loop_stayed_free, + timed_with_loop_lags, + warm_tokenizer, +) from tests.test_litellm.litellm_core_utils.messages_with_counts import ( MESSAGES_TEXT, MESSAGES_WITH_IMAGES, @@ -127,30 +132,15 @@ def test_valid_chunk_size_config_is_honoured(monkeypatch): importlib.reload(litellm.constants) -async def _loop_wake_lags(until: asyncio.Event) -> tuple[float, ...]: - async def wake_lag() -> float: - started: Final = time.perf_counter() - await asyncio.sleep(0.001) - return time.perf_counter() - started - 0.001 - - return tuple([await wake_lag() for _ in iter(until.is_set, True)]) - - async def test_huggingface_count_in_a_worker_thread_leaves_the_event_loop_free(): - counted: Final = asyncio.Event() + warm_tokenizer("claude-fable-5") - async def count_off_the_loop() -> tuple[int, float]: - started: Final = time.perf_counter() - try: - tokens: Final = await asyncify(token_counter_new)(model="claude-fable-5", text=text * 100) - return tokens, time.perf_counter() - started - finally: - counted.set() - - (tokens, took), lags = await asyncio.gather(count_off_the_loop(), _loop_wake_lags(counted)) + tokens, took, lags = await timed_with_loop_lags( + lambda: asyncify(token_counter_new)(model="claude-fable-5", text=text * 100) + ) assert tokens > 0 - assert max(lags) < took / 4, f"the event loop stalled {max(lags):.3f}s during a {took:.3f}s count" + assert_loop_stayed_free(took, lags) @pytest.mark.parametrize("max_exact_chars", [64, 1_000, 2_500]) diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py index 2839acab6b0..398abf60b54 100644 --- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -3670,5 +3670,42 @@ async def test_post_call_success_hook_contains_header_merge_failures( ) +@pytest.mark.asyncio +async def test_the_project_itpm_reservation_counts_the_request_off_the_event_loop(rate_limiter): + from tests.large_text import text + from tests.test_litellm.litellm_core_utils.event_loop_lag import ( + assert_loop_stayed_free, + timed_with_loop_lags, + warm_tokenizer, + ) + + handler, _cache = rate_limiter + stash = get_or_create_request_stash() + warm_tokenizer("claude-fable-5") + data: Dict[str, Any] = { + "model": "claude-fable-5", + "messages": [{"role": "user", "content": text * 100}], + } + itpm_descriptor = { + "key": PROJECT_ITPM_DESCRIPTOR_KEY, + "value": "proj-loop:claude-fable-5", + "rate_limit": {"tokens_per_unit": 10_000_000, "window_size": 60}, + } + + _, took, lags = await timed_with_loop_lags( + lambda: handler._reserve_project_io_tokens_or_raise( + descriptors=[itpm_descriptor], + data=data, + requested_model="claude-fable-5", + user_api_key_dict=UserAPIKeyAuth(api_key=hash_token("sk-itpm-loop"), project_id="proj-loop"), + tpm_reservation_scopes=[], + tpm_reservation_amount=0, + ) + ) + + assert stash.rate_limit_response is not None + assert_loop_stayed_free(took, lags) + + if __name__ == "__main__": pytest.main([__file__, "-v", "-s"]) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 20b41cf1530..7fa3d5dd246 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -12860,32 +12860,20 @@ async def test_update_general_settings_keeps_yaml_openai_websocket_passthrough() assert ps.general_settings["enable_openai_websocket_passthrough"] is False -async def _loop_wake_lags(until: asyncio.Event) -> tuple[float, ...]: - async def wake_lag() -> float: - started: Final = time.perf_counter() - await asyncio.sleep(0.001) - return time.perf_counter() - started - 0.001 - - return tuple([await wake_lag() for _ in iter(until.is_set, True)]) - - async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_count(monkeypatch): from tests.large_text import text + from tests.test_litellm.litellm_core_utils.event_loop_lag import ( + assert_loop_stayed_free, + timed_with_loop_lags, + warm_tokenizer, + ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) - counted: Final = asyncio.Event() + warm_tokenizer("claude-fable-5") - async def count_off_the_loop() -> tuple[int, float]: - started: Final = time.perf_counter() - try: - response: Final = await proxy_server_module.token_counter( - TokenCountRequest(model="claude-fable-5", prompt=text * 100) - ) - return response.total_tokens, time.perf_counter() - started - finally: - counted.set() + response, took, lags = await timed_with_loop_lags( + lambda: proxy_server_module.token_counter(TokenCountRequest(model="claude-fable-5", prompt=text * 100)) + ) - (tokens, took), lags = await asyncio.gather(count_off_the_loop(), _loop_wake_lags(counted)) - - assert tokens > 0 - assert max(lags) < took / 4, f"the event loop stalled {max(lags):.3f}s during a {took:.3f}s count" + assert response.total_tokens > 0 + assert_loop_stayed_free(took, lags) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 9462f2c8eb0..b78ec7dcff6 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -1823,6 +1823,44 @@ def test_a_dispatched_failure_lifts_the_four_fields_the_spend_log_needs(): assert lifted["standard_logging_object"] == {"id": "log-1"} +@pytest.mark.asyncio +async def test_a_dispatched_failure_is_counted_off_the_event_loop(): + from unittest.mock import AsyncMock, patch + + from tests.large_text import text + from tests.test_litellm.litellm_core_utils.event_loop_lag import ( + assert_loop_stayed_free, + timed_with_loop_lags, + warm_tokenizer, + ) + + warm_tokenizer("claude-fable-5") + request_data = { + "litellm_logging_obj": _LoggingObj( + { + "first_api_call_start_time": 1700000000.0, + "call_type": "acompletion", + "model": "claude-fable-5", + "messages": [{"role": "user", "content": text * 100}], + } + ), + "metadata": {}, + } + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): + _, took, lags = await timed_with_loop_lags( + lambda: proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=Exception("boom"), + user_api_key_dict=UserAPIKeyAuth(), + ) + ) + + assert request_data["combined_usage_object"].prompt_tokens > 0 + assert_loop_stayed_free(took, lags) + + @pytest.mark.asyncio async def test_proxy_only_error_expected_4xx_skips_traceback_for_both_handlers(monkeypatch): """Regression for LIT-6043: an expected 4xx must not format a traceback for From 415bdbfd8f6ba9bd0422087cc34509bde962ce55 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:46:39 -0700 Subject: [PATCH 082/411] fix(azure_ai): charge the Model Router fee once and correct catalog limits The router fee was folded into azure_ai.cost_per_token and then added again by the additional_costs hook, so every routed request paid it twice. The hook now owns the fee, the entry named by the deployment supplies the price, and a response priced as the router entry itself is not charged again model-router, gpt-chat-latest and cohere-command-a carry the limits from the Foundry models page, and model-router and grok-4-20-* carry their retirement dates. The router tests now run at the completion_cost level with a Logging object, which is the path the proxy takes, and fail at the merge base --- litellm/cost_calculator.py | 9 +- litellm/llms/azure_ai/cost_calculator.py | 87 ++- ...odel_prices_and_context_window_backup.json | 11 +- model_prices_and_context_window.json | 11 +- .../azure_ai/test_azure_ai_cost_calculator.py | 561 ++++++------------ ...azure_ai_foundry_catalog_model_metadata.py | 30 +- 6 files changed, 246 insertions(+), 463 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9a9d2ceda03..fc896a098b3 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -45,6 +45,9 @@ from litellm.llms.azure.cost_calculation import ( from litellm.llms.azure_ai.cost_calculator import ( cost_per_token as azure_ai_cost_per_token, ) +from litellm.llms.azure_ai.cost_calculator import ( + is_router_fee_entry as azure_ai_is_router_fee_entry, +) from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.llms.bedrock.cost_calculation import ( cost_per_token as bedrock_cost_per_token, @@ -338,8 +341,6 @@ def cost_per_token( ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") response: Any | None = None, - ### REQUEST MODEL ### - request_model: str | None = None, # original request model for router detection ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -661,7 +662,6 @@ def cost_per_token( model=model, usage=usage_block, response_time_ms=response_time_ms, - request_model=request_model, service_tier=service_tier, ) else: @@ -1659,11 +1659,10 @@ def completion_cost( data_residency=data_residency, vertex_location=vertex_location, response=completion_response, - request_model=request_model_for_cost, ) # Get additional costs from provider (e.g., routing fees, infrastructure costs) - if custom_llm_provider == "azure_ai": + if custom_llm_provider == "azure_ai" and not azure_ai_is_router_fee_entry(model): model_for_additional_costs = request_model_for_cost if completion_response is not None: hidden_params = getattr(completion_response, "_hidden_params", None) or {} diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 141148f06e7..8a48860c0c4 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -31,6 +31,18 @@ def _is_azure_model_router(model: str) -> bool: return "model-router" in model_lower or "model_router" in model_lower or model_lower == "azure-model-router" +ROUTER_FEE_ENTRY_NAMES: Final = frozenset({"model-router", "model_router"}) + + +def is_router_fee_entry(model: str) -> bool: + return model.lower().removeprefix("azure_ai/") in ROUTER_FEE_ENTRY_NAMES + + +def _router_fee_entry_name(model: str) -> str: + entry_name: Final = model.lower().removeprefix("azure_ai/") + return entry_name if entry_name in ROUTER_FEE_ENTRY_NAMES else "model_router" + + def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> float: """ Calculate the flat cost for Azure AI Foundry Model Router. @@ -44,26 +56,39 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl """ if not _is_azure_model_router(model): return 0.0 - - # Get the model router pricing from model_prices_and_context_window.json - # Use "model_router" as the key (without actual model name suffix) - model_info: Final = get_model_info(model="model_router", custom_llm_provider="azure_ai") + model_info: Final = get_model_info(model=_router_fee_entry_name(model), custom_llm_provider="azure_ai") router_flat_cost_per_token: Final = model_info.get("input_cost_per_token", 0) - if router_flat_cost_per_token and router_flat_cost_per_token > 0: return prompt_tokens * router_flat_cost_per_token - return 0.0 -ROUTER_FEE_ENTRY_NAMES: Final = frozenset({"model-router", "model_router"}) +def cost_per_token( + model: str, + usage: Usage, + response_time_ms: float | None = 0.0, + service_tier: str | None = None, +) -> tuple[float, float]: + """ + Price the response model's own tokens for Azure AI. + The Azure AI Foundry Model Router fee is not part of this: completion_cost charges it once through + AzureModelRouterConfig.calculate_additional_costs as the "Azure Model Router Flat Cost" line of the cost + breakdown, and a response priced as the router entry itself already carries it. A router deployment name + that is missing from the cost map prices at zero here so that line item is the whole cost. -def _prices_router_fee_itself(model: str) -> bool: - return model.lower().rsplit("/", 1)[-1] in ROUTER_FEE_ENTRY_NAMES + Args: + model: str, the model name without provider prefix (from response) + usage: LiteLLM Usage block + response_time_ms: Optional response time in milliseconds + service_tier: Optional service tier the request was priced on + Returns: + Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd -def _base_cost_per_token(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float] | None: + Raises: + ValueError: If a model that is not a Model Router name is missing from the cost map + """ try: return generic_cost_per_token( model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier @@ -72,44 +97,6 @@ def _base_cost_per_token(model: str, usage: Usage, service_tier: str | None) -> if not _is_azure_model_router(model): raise verbose_logger.debug( - "Azure AI Model Router: model '%s' not in cost map, calculating routing flat cost only. Error: %s", model, e + "Azure AI Model Router: model '%s' not in cost map, only the routing fee applies. Error: %s", model, e ) - return None - - -def cost_per_token( - model: str, - usage: Usage, - response_time_ms: float | None = 0.0, - request_model: str | None = None, - service_tier: str | None = None, -) -> tuple[float, float]: - """ - Calculate the cost per token for Azure AI models. - - For Azure AI Foundry Model Router the routing fee (the azure_ai/model_router entry, $0.14 per - million input tokens) is added on top of the routed model's cost. When the response model is - the router entry itself, generic_cost_per_token has already charged that fee. - - Args: - model: str, the model name without provider prefix (from response) - usage: LiteLLM Usage block - response_time_ms: Optional response time in milliseconds - request_model: Optional[str], the original request model name (to detect router usage) - - Returns: - Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd - - Raises: - ValueError: If the model is not found in the cost map and cost cannot be calculated - (except for Model Router models where we return just the routing flat cost) - """ - is_router_request: Final = _is_azure_model_router(model) or ( - request_model is not None and _is_azure_model_router(request_model) - ) - base_cost: Final = _base_cost_per_token(model=model, usage=usage, service_tier=service_tier) - prompt_cost, completion_cost = base_cost if base_cost is not None else (0.0, 0.0) - if not is_router_request or (base_cost is not None and _prices_router_fee_itself(model)): - return prompt_cost, completion_cost - router_flat_cost: Final = calculate_azure_model_router_flat_cost(request_model or model, usage.prompt_tokens) - return prompt_cost + router_flat_cost, completion_cost + return 0.0, 0.0 diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6649fa831d7..483ebd2431c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3586,7 +3586,7 @@ "deprecation_date": "2026-12-02", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 200000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -4068,10 +4068,11 @@ "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, "azure_ai/model-router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", - "max_input_tokens": 1048576, + "max_input_tokens": 200000, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -10393,8 +10394,8 @@ "input_cost_per_token": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 8182, + "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", @@ -10753,6 +10754,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -10769,6 +10771,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6649fa831d7..483ebd2431c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3586,7 +3586,7 @@ "deprecation_date": "2026-12-02", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", - "max_input_tokens": 200000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -4068,10 +4068,11 @@ "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, "azure_ai/model-router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", - "max_input_tokens": 1048576, + "max_input_tokens": 200000, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -10393,8 +10394,8 @@ "input_cost_per_token": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 8182, + "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", @@ -10753,6 +10754,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -10769,6 +10771,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 80cd99bd46b..20d0ec03a2a 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -2,20 +2,25 @@ Test Azure AI cost calculator, especially Model Router flat cost. """ +from datetime import datetime +from typing import Final + import pytest +import litellm +from litellm.cost_calculator import completion_cost +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.cost_calculator import ( _is_azure_model_router, + calculate_azure_model_router_flat_cost, cost_per_token, ) -from litellm.types.utils import Usage +from litellm.types.utils import Choices, Message, ModelResponse, Usage from litellm.utils import get_model_info # Get the flat cost from model_prices_and_context_window.json _model_info = get_model_info(model="model_router", custom_llm_provider="azure_ai") -AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS = ( - _model_info.get("input_cost_per_token", 0) * 1_000_000 -) +AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS = _model_info.get("input_cost_per_token", 0) * 1_000_000 class TestAzureModelRouterDetection: @@ -80,377 +85,172 @@ class TestAzureModelRouterPrefix: assert result == expected +ROUTER_FEE_PER_TOKEN: Final = AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 +ROUTED_MODEL: Final = "gpt-4.1-nano-2025-04-14" +ROUTED_USAGE: Final = Usage(prompt_tokens=5000, completion_tokens=2000, total_tokens=7000) +ROUTED_FEE: Final = 5000 * ROUTER_FEE_PER_TOKEN + + +def _router_logging(request_model: str) -> Logging: + return Logging( + model=request_model, + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="test-123", + function_id="test-function", + ) + + +def _azure_ai_response(response_model: str, litellm_model_name: str | None = None) -> ModelResponse: + response: Final = ModelResponse( + id="test-123", + choices=[Choices(finish_reason="stop", index=0, message=Message(role="assistant", content="Hello"))], + created=1234567890, + model=response_model, + object="chat.completion", + usage=ROUTED_USAGE, + ) + response._hidden_params = ( + {"custom_llm_provider": "azure_ai"} + if litellm_model_name is None + else {"custom_llm_provider": "azure_ai", "litellm_model_name": litellm_model_name} + ) + return response + + +def _routed_model_cost() -> tuple[float, float]: + routed_info: Final = get_model_info(model=ROUTED_MODEL, custom_llm_provider="azure_ai") + return ( + ROUTED_USAGE.prompt_tokens * (routed_info["input_cost_per_token"] or 0.0), + ROUTED_USAGE.completion_tokens * (routed_info["output_cost_per_token"] or 0.0), + ) + + +@pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterFlatCost: - """Test Azure AI Foundry Model Router flat cost calculation.""" + """cost_per_token prices the response model only; the router fee is the cost breakdown's own line item.""" - def test_model_router_flat_cost_basic(self): - """Test that flat cost is added for Model Router requests.""" - model = "azure-model-router" - usage = Usage( - prompt_tokens=1000, - completion_tokens=500, - total_tokens=1500, + def test_unmapped_router_deployment_name_prices_at_zero(self) -> None: + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + assert cost_per_token(model="azure-model-router", usage=usage) == (0.0, 0.0) + + @pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) + def test_router_entry_prices_its_own_fee(self, router_entry_name: str) -> None: + usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) + prompt_cost, completion_cost_usd = cost_per_token(model=router_entry_name, usage=usage) + assert prompt_cost == pytest.approx(0.14, rel=1e-9) + assert completion_cost_usd == 0.0 + + def test_routed_model_is_priced_as_itself(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + prompt_cost, completion_cost_usd = cost_per_token(model=ROUTED_MODEL, usage=ROUTED_USAGE) + assert routed_prompt_cost > 0 + assert prompt_cost == pytest.approx(routed_prompt_cost, rel=1e-9) + assert completion_cost_usd == pytest.approx(routed_completion_cost, rel=1e-9) + + def test_unmapped_model_that_is_not_a_router_name_raises(self) -> None: + usage = Usage(prompt_tokens=10, completion_tokens=10, total_tokens=20) + with pytest.raises(Exception, match="no-such-azure-ai-model"): + cost_per_token(model="no-such-azure-ai-model", usage=usage) + + def test_flat_cost_helper(self) -> None: + assert calculate_azure_model_router_flat_cost( + model="azure-model-router", prompt_tokens=10_000 + ) == pytest.approx(0.0014, rel=1e-9) + assert calculate_azure_model_router_flat_cost(model="gpt-5-nano", prompt_tokens=10_000) == 0.0 + + def test_flat_cost_reads_the_fee_from_the_deployment_named_entry(self) -> None: + litellm.register_model( + {"azure_ai/model-router": {"input_cost_per_token": 2e-07, "litellm_provider": "azure_ai", "mode": "chat"}} ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # Calculate expected flat cost - expected_flat_cost = ( - usage.prompt_tokens - * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - / 1_000_000 + litellm.get_model_info.cache_clear() + assert calculate_azure_model_router_flat_cost(model="model-router", prompt_tokens=1_000_000) == pytest.approx( + 0.2, rel=1e-9 ) - - # Flat cost should be $0.00014 (1000 tokens × $0.14 / 1M tokens) - assert expected_flat_cost == pytest.approx(0.00014, rel=1e-9) - - # Prompt cost should include the flat cost - # (plus any base cost from the actual model used, which might be 0 if not in model_cost) - assert prompt_cost >= expected_flat_cost - print( - f"Model Router flat cost for {usage.prompt_tokens} tokens: ${expected_flat_cost:.6f}" - ) - print(f"Total prompt cost: ${prompt_cost:.6f}") - - def test_model_router_flat_cost_large_request(self): - """Test flat cost calculation for larger requests.""" - model = "model-router" - usage = Usage( - prompt_tokens=100_000, - completion_tokens=50_000, - total_tokens=150_000, - ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # Calculate expected flat cost - expected_flat_cost = ( - usage.prompt_tokens - * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - / 1_000_000 - ) - - # Flat cost should be $0.014 (100k tokens × $0.14 / 1M tokens) - assert expected_flat_cost == pytest.approx(0.014, rel=1e-9) - # Use approx for floating-point comparison - assert prompt_cost >= expected_flat_cost or prompt_cost == pytest.approx( - expected_flat_cost, rel=1e-9 - ) - print( - f"Model Router flat cost for {usage.prompt_tokens} tokens: ${expected_flat_cost:.6f}" - ) - print(f"Total prompt cost: ${prompt_cost:.6f}") - - def test_model_router_flat_cost_1m_tokens(self): - """Test flat cost for exactly 1 million input tokens.""" - model = "azure-model-router" - usage = Usage( - prompt_tokens=1_000_000, - completion_tokens=100_000, - total_tokens=1_100_000, - ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # Calculate expected flat cost - expected_flat_cost = AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - - # Flat cost should be exactly $0.14 for 1M tokens - assert expected_flat_cost == pytest.approx(0.14, rel=1e-9) - assert prompt_cost >= expected_flat_cost - print(f"Model Router flat cost for 1M tokens: ${expected_flat_cost:.6f}") - print(f"Total prompt cost: ${prompt_cost:.6f}") - - def test_non_model_router_no_flat_cost(self): - """Test that non-Model Router models don't get the flat cost.""" - model = "gpt-4o" - usage = Usage( - prompt_tokens=1000, - completion_tokens=500, - total_tokens=1500, - ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # No flat cost should be added for non-Model Router models - # The cost might be 0 or based on the model's pricing - print(f"Non-Model Router prompt cost: ${prompt_cost:.6f}") - # We just ensure it doesn't crash and returns valid values - assert prompt_cost >= 0 - assert completion_cost >= 0 - - def test_model_router_with_cached_tokens(self): - """Test Model Router flat cost with cached tokens.""" - model = "azure-model-router" - usage = Usage( - prompt_tokens=2000, - completion_tokens=800, - total_tokens=2800, - cache_read_input_tokens=500, - cache_creation_input_tokens=200, - ) - - prompt_cost, completion_cost = cost_per_token(model=model, usage=usage) - - # Flat cost is based on ALL prompt tokens (including cached) - expected_flat_cost = ( - usage.prompt_tokens - * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - / 1_000_000 - ) - - assert expected_flat_cost == pytest.approx(0.00028, rel=1e-9) - assert prompt_cost >= expected_flat_cost - print( - f"Model Router flat cost with caching for {usage.prompt_tokens} tokens: ${expected_flat_cost:.6f}" - ) - print(f"Total prompt cost: ${prompt_cost:.6f}") - - def test_router_flat_cost_when_response_has_actual_model(self): - """ - Test that router flat cost is added when request was via router but response - contains the actual model (e.g., gpt-5-nano). - - This is the key fix: Azure returns the actual model in the response, but we - must still add the router flat cost because the request was made via model router. - """ - usage = Usage( - prompt_tokens=10000, - completion_tokens=5000, - total_tokens=15000, - ) - - # Response model is the actual model Azure used (not a router name) - response_model = "gpt-5-nano-2025-08-07" - # Request model is the router - user called azure_ai/model_router/model-router - request_model = "azure_ai/model_router/model-router" - - prompt_cost, completion_cost = cost_per_token( - model=response_model, - usage=usage, - request_model=request_model, - ) - - # Expected: model cost (from gpt-5-nano) + router flat cost - expected_flat_cost = ( - usage.prompt_tokens - * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS - / 1_000_000 - ) - assert expected_flat_cost == pytest.approx(0.0014, rel=1e-9) - - # Total cost should be model cost + flat cost - total_cost = prompt_cost + completion_cost - assert total_cost >= expected_flat_cost - - # Prompt cost should include both model prompt cost and router flat cost - assert prompt_cost >= expected_flat_cost + assert calculate_azure_model_router_flat_cost( + model="azure-model-router", prompt_tokens=1_000_000 + ) == pytest.approx(0.14, rel=1e-9) +@pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterCostBreakdown: - """Test that Azure Model Router flat cost is tracked in cost breakdown.""" + """completion_cost charges the router fee exactly once, as the cost breakdown's additional cost line.""" - def test_flat_cost_calculation_helper(self): - """Test that flat cost can be calculated using the helper function.""" - from litellm.llms.azure_ai.cost_calculator import ( - calculate_azure_model_router_flat_cost, - ) - - model = "azure-model-router" - prompt_tokens = 10000 - - # Calculate flat cost using helper function - flat_cost = calculate_azure_model_router_flat_cost( - model=model, prompt_tokens=prompt_tokens - ) - - # Expected flat cost - expected_flat_cost = ( - prompt_tokens * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 - ) - - assert flat_cost > 0 - assert flat_cost == pytest.approx(expected_flat_cost, rel=1e-9) - print(f"Flat cost calculated: ${flat_cost:.6f}") - - def test_flat_cost_integration_with_completion_cost(self): - """Test that flat cost is properly integrated into completion_cost calculation.""" - import litellm - from litellm.cost_calculator import completion_cost - from litellm.types.utils import Choices, Message, ModelResponse, Usage - - # Create a mock response for azure_ai model router - response = ModelResponse( - id="test-123", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - role="assistant", - content="Test response", - ), - ) - ], - created=1234567890, - model="azure-model-router", - object="chat.completion", - usage=Usage( - prompt_tokens=5000, - completion_tokens=2000, - total_tokens=7000, - ), - ) - - # Set hidden params for provider - response._hidden_params = {"custom_llm_provider": "azure_ai"} - - # Calculate cost + def test_unmapped_router_deployment_name_costs_only_the_fee(self) -> None: cost = completion_cost( - completion_response=response, + completion_response=_azure_ai_response("azure-model-router"), model="azure-model-router", custom_llm_provider="azure_ai", ) + assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) - # Expected flat cost - expected_flat_cost = ( - 5000 * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 - ) - - # Cost should include the flat cost (use approx for floating-point comparison) - assert cost >= expected_flat_cost or cost == pytest.approx( - expected_flat_cost, rel=1e-9 - ) - print(f"Total cost with flat fee: ${cost:.6f}") - print(f"Expected minimum flat cost: ${expected_flat_cost:.6f}") - - def test_additional_costs_in_cost_breakdown(self): - """Test that Azure Model Router flat cost appears in additional_costs dict.""" - from datetime import datetime - - from litellm.cost_calculator import completion_cost - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import Choices, Message, ModelResponse, Usage - - # Create logging object with required parameters - logging_obj = Logging( - model="azure-model-router", - messages=[{"role": "user", "content": "Hello"}], - stream=False, - call_type="completion", - start_time=datetime.now(), - litellm_call_id="test-123", - function_id="test-function", - ) - - # Create a mock response for azure_ai model router - response = ModelResponse( - id="test-123", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - role="assistant", - content="Test response", - ), - ) - ], - created=1234567890, - model="azure-model-router", - object="chat.completion", - usage=Usage( - prompt_tokens=5000, - completion_tokens=2000, - total_tokens=7000, - ), - ) - - # Set hidden params for provider - response._hidden_params = {"custom_llm_provider": "azure_ai"} - - # Calculate cost with logging object + def test_fee_is_the_breakdown_line_item_for_an_unmapped_router_name(self) -> None: + logging_obj = _router_logging("azure-model-router") cost = completion_cost( - completion_response=response, + completion_response=_azure_ai_response("azure-model-router"), model="azure-model-router", custom_llm_provider="azure_ai", litellm_logging_obj=logging_obj, ) - - # Check that cost breakdown contains additional_costs - assert hasattr(logging_obj, "cost_breakdown") - assert logging_obj.cost_breakdown is not None - assert "additional_costs" in logging_obj.cost_breakdown - assert isinstance(logging_obj.cost_breakdown["additional_costs"], dict) - - # Check that the Azure Model Router flat cost is in additional_costs - additional_costs = logging_obj.cost_breakdown["additional_costs"] - assert "Azure Model Router Flat Cost" in additional_costs - - # Verify the flat cost value - expected_flat_cost = ( - 5000 * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + assert breakdown["input_cost"] == 0.0 + assert breakdown.get("additional_costs") == pytest.approx( + {"Azure Model Router Flat Cost": ROUTED_FEE}, rel=1e-9 ) - actual_flat_cost = additional_costs["Azure Model Router Flat Cost"] - assert actual_flat_cost == pytest.approx(expected_flat_cost, rel=1e-9) + assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) - print(f"Additional costs in breakdown: {additional_costs}") - print(f"Azure Model Router Flat Cost: ${actual_flat_cost:.6f}") - - def test_additional_costs_when_response_has_actual_model_via_hidden_params(self): - """additional_costs populated when response has actual model but request was via model router (hidden_params).""" - from datetime import datetime - - from litellm.cost_calculator import completion_cost - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import Choices, Message, ModelResponse, Usage - - logging_obj = Logging( - model="gpt-4.1-nano-2025-04-14", - messages=[{"role": "user", "content": "Hello"}], - stream=False, - call_type="completion", - start_time=datetime.now(), - litellm_call_id="test-123", - function_id="test-function", - ) - response = ModelResponse( - id="test-123", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message(role="assistant", content="Hello"), - ) - ], - created=1234567890, - model="gpt-4.1-nano-2025-04-14", - object="chat.completion", - usage=Usage(prompt_tokens=5000, completion_tokens=2000, total_tokens=7000), - ) - response._hidden_params = { - "custom_llm_provider": "azure_ai", - "litellm_model_name": "azure_ai/model-router", - } + def test_router_request_with_routed_response_charges_the_fee_once(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + logging_obj = _router_logging("model-router") cost = completion_cost( - completion_response=response, - model="gpt-4.1-nano-2025-04-14", + completion_response=_azure_ai_response(ROUTED_MODEL), + model=ROUTED_MODEL, custom_llm_provider="azure_ai", litellm_logging_obj=logging_obj, ) - expected_flat_cost = ( - 5000 * AZURE_MODEL_ROUTER_FLAT_COST_PER_M_INPUT_TOKENS / 1_000_000 + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + assert breakdown["input_cost"] == pytest.approx(routed_prompt_cost, rel=1e-9) + assert breakdown["output_cost"] == pytest.approx(routed_completion_cost, rel=1e-9) + assert breakdown.get("additional_costs") == pytest.approx( + {"Azure Model Router Flat Cost": ROUTED_FEE}, rel=1e-9 ) - assert cost >= expected_flat_cost - assert logging_obj.cost_breakdown is not None - assert "additional_costs" in logging_obj.cost_breakdown - assert ( - "Azure Model Router Flat Cost" - in logging_obj.cost_breakdown["additional_costs"] + assert cost == pytest.approx(routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9) + + def test_routed_response_named_by_hidden_params_charges_the_fee_once(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + logging_obj = _router_logging(ROUTED_MODEL) + cost = completion_cost( + completion_response=_azure_ai_response(ROUTED_MODEL, litellm_model_name="azure_ai/model-router"), + model=ROUTED_MODEL, + custom_llm_provider="azure_ai", + litellm_logging_obj=logging_obj, ) - assert logging_obj.cost_breakdown["additional_costs"][ - "Azure Model Router Flat Cost" - ] == pytest.approx(expected_flat_cost, rel=1e-9) + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + assert breakdown["input_cost"] == pytest.approx(routed_prompt_cost, rel=1e-9) + assert breakdown.get("additional_costs") == pytest.approx( + {"Azure Model Router Flat Cost": ROUTED_FEE}, rel=1e-9 + ) + assert cost == pytest.approx(routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9) + + @pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) + def test_response_priced_as_the_router_entry_charges_the_fee_once(self, router_entry_name: str) -> None: + logging_obj = _router_logging(router_entry_name) + cost = completion_cost( + completion_response=_azure_ai_response(router_entry_name), + model=router_entry_name, + custom_llm_provider="azure_ai", + litellm_logging_obj=logging_obj, + ) + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + assert "additional_costs" not in breakdown + assert breakdown["input_cost"] == pytest.approx(ROUTED_FEE, rel=1e-9) + assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) class TestAzureAIServiceTierCostCalculation: @@ -459,26 +259,27 @@ class TestAzureAIServiceTierCostCalculation: @pytest.fixture(autouse=True) def register_test_model(self): import litellm - litellm.register_model(model_cost={ - "test-azure-ai-model": { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "input_cost_per_token_priority": 0.01, - "output_cost_per_token_priority": 0.02, - "input_cost_per_token_flex": 0.0005, - "output_cost_per_token_flex": 0.001, - "litellm_provider": "azure_ai", - "max_tokens": 8192, + + litellm.register_model( + model_cost={ + "test-azure-ai-model": { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "input_cost_per_token_priority": 0.01, + "output_cost_per_token_priority": 0.02, + "input_cost_per_token_flex": 0.0005, + "output_cost_per_token_flex": 0.001, + "litellm_provider": "azure_ai", + "max_tokens": 8192, + } } - }) + ) def test_service_tier_priority_higher_cost(self): """Priority tier should cost more than standard for azure_ai.""" usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - standard_prompt, standard_completion = cost_per_token( - model="test-azure-ai-model", usage=usage - ) + standard_prompt, standard_completion = cost_per_token(model="test-azure-ai-model", usage=usage) priority_prompt, priority_completion = cost_per_token( model="test-azure-ai-model", usage=usage, service_tier="priority" ) @@ -490,12 +291,8 @@ class TestAzureAIServiceTierCostCalculation: """Flex tier should cost less than standard for azure_ai.""" usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - standard_prompt, standard_completion = cost_per_token( - model="test-azure-ai-model", usage=usage - ) - flex_prompt, flex_completion = cost_per_token( - model="test-azure-ai-model", usage=usage, service_tier="flex" - ) + standard_prompt, standard_completion = cost_per_token(model="test-azure-ai-model", usage=usage) + flex_prompt, flex_completion = cost_per_token(model="test-azure-ai-model", usage=usage, service_tier="flex") assert flex_prompt < standard_prompt assert flex_completion < standard_completion @@ -528,29 +325,3 @@ def test_mai_thinking_1_model_info_and_cost(local_model_cost_map): assert model_info["supports_function_calling"] is True assert prompt_cost == pytest.approx(2.0) assert completion_cost == pytest.approx(8.0) - - -@pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) -def test_router_entry_as_response_model_charges_the_fee_once(router_entry_name: str) -> None: - usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) - prompt_cost, completion_cost = cost_per_token(model=router_entry_name, usage=usage) - assert prompt_cost == pytest.approx(0.14, rel=1e-9) - assert completion_cost == 0.0 - - -@pytest.mark.usefixtures("local_model_cost_map") -def test_unmapped_router_deployment_name_still_charges_the_fee() -> None: - usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) - prompt_cost, completion_cost = cost_per_token(model="azure-model-router", usage=usage) - assert prompt_cost == pytest.approx(0.14, rel=1e-9) - assert completion_cost == 0.0 - - -@pytest.mark.usefixtures("local_model_cost_map") -def test_routed_model_response_adds_the_fee_on_top() -> None: - usage = Usage(prompt_tokens=1_000_000, completion_tokens=0, total_tokens=1_000_000) - routed_prompt_cost, _ = cost_per_token(model="gpt-5-nano", usage=usage) - prompt_cost, _ = cost_per_token(model="gpt-5-nano", usage=usage, request_model="azure_ai/model-router") - assert routed_prompt_cost > 0 - assert prompt_cost == pytest.approx(routed_prompt_cost + 0.14, rel=1e-9) diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 1b4e83438a6..fab9be1b42c 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -26,6 +26,7 @@ class TokenPricedCatalogModel: max_input_tokens: int max_output_tokens: int cache_read_input_token_cost: float | None + deprecation_date: str | None supported_flags: tuple[str, ...] @@ -36,9 +37,10 @@ TOKEN_PRICED_MODELS: Final = ( source=AZURE_OPENAI_PRICING, input_cost_per_token=5e-06, output_cost_per_token=3e-05, - max_input_tokens=200000, + max_input_tokens=272000, max_output_tokens=128000, cache_read_input_token_cost=5e-07, + deprecation_date="2026-12-02", supported_flags=( "supports_function_calling", "supports_prompt_caching", @@ -58,7 +60,13 @@ TOKEN_PRICED_MODELS: Final = ( max_input_tokens=200000, max_output_tokens=100000, cache_read_input_token_cost=3.75e-07, - supported_flags=("supports_function_calling", "supports_prompt_caching", "supports_reasoning", "supports_vision"), + deprecation_date="2026-11-15", + supported_flags=( + "supports_function_calling", + "supports_prompt_caching", + "supports_reasoning", + "supports_vision", + ), ), TokenPricedCatalogModel( catalog_name="model-router", @@ -66,9 +74,10 @@ TOKEN_PRICED_MODELS: Final = ( source=FOUNDRY_AOAI_PRICING, input_cost_per_token=1.4e-07, output_cost_per_token=0.0, - max_input_tokens=1048576, + max_input_tokens=200000, max_output_tokens=32768, cache_read_input_token_cost=None, + deprecation_date="2027-05-20", supported_flags=(), ), TokenPricedCatalogModel( @@ -78,8 +87,9 @@ TOKEN_PRICED_MODELS: Final = ( input_cost_per_token=2.5e-06, output_cost_per_token=1e-05, max_input_tokens=131072, - max_output_tokens=4096, + max_output_tokens=8182, cache_read_input_token_cost=None, + deprecation_date=None, supported_flags=("supports_function_calling", "supports_tool_choice"), ), TokenPricedCatalogModel( @@ -91,6 +101,7 @@ TOKEN_PRICED_MODELS: Final = ( max_input_tokens=262000, max_output_tokens=8192, cache_read_input_token_cost=None, + deprecation_date="2027-04-06", supported_flags=( "supports_function_calling", "supports_reasoning", @@ -109,6 +120,7 @@ TOKEN_PRICED_MODELS: Final = ( max_input_tokens=262000, max_output_tokens=8192, cache_read_input_token_cost=None, + deprecation_date="2027-04-06", supported_flags=( "supports_function_calling", "supports_response_schema", @@ -146,7 +158,9 @@ def test_azure_ai_catalog_name_is_priced_and_routed(spec: TokenPricedCatalogMode @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize( - "spec", [spec for spec in TOKEN_PRICED_MODELS if spec.catalog_name != "model-router"], ids=lambda spec: spec.catalog_name + "spec", + [spec for spec in TOKEN_PRICED_MODELS if spec.catalog_name != "model-router"], + ids=lambda spec: spec.catalog_name, ) def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: prompt_cost, completion_cost = cost_per_token( @@ -174,3 +188,9 @@ def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> No assert str(main_entry["source"]).startswith("https://azure.microsoft.com/en-us/pricing/details/") assert backup_entry == main_entry + + +@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) +def test_azure_ai_catalog_entry_carries_its_retirement_date(spec: TokenPricedCatalogModel) -> None: + entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", spec.catalog_name) + assert entry.get("deprecation_date") == spec.deprecation_date From c02f2dc0feff1f95d61b1be699565a835402a122 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:10:54 -0700 Subject: [PATCH 083/411] fix(azure_ai): keep the request_model keyword on cost_per_token Restores the public keyword removed at 415bdbfd8f. A direct caller that names the Model Router as the request model gets the routing fee folded into the prompt cost once; completion_cost never passes it and charges the fee through the additional-costs hook as before --- litellm/cost_calculator.py | 3 + litellm/llms/azure_ai/cost_calculator.py | 62 +++++++++++-------- .../azure_ai/test_azure_ai_cost_calculator.py | 33 ++++++++++ 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index fc896a098b3..e135503d11d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -341,6 +341,8 @@ def cost_per_token( ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") response: Any | None = None, + ### REQUEST MODEL ### + request_model: str | None = None, # original request model for router detection ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -662,6 +664,7 @@ def cost_per_token( model=model, usage=usage_block, response_time_ms=response_time_ms, + request_model=request_model, service_tier=service_tier, ) else: diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 8a48860c0c4..e57ba055587 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -63,32 +63,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl return 0.0 -def cost_per_token( - model: str, - usage: Usage, - response_time_ms: float | None = 0.0, - service_tier: str | None = None, -) -> tuple[float, float]: - """ - Price the response model's own tokens for Azure AI. - - The Azure AI Foundry Model Router fee is not part of this: completion_cost charges it once through - AzureModelRouterConfig.calculate_additional_costs as the "Azure Model Router Flat Cost" line of the cost - breakdown, and a response priced as the router entry itself already carries it. A router deployment name - that is missing from the cost map prices at zero here so that line item is the whole cost. - - Args: - model: str, the model name without provider prefix (from response) - usage: LiteLLM Usage block - response_time_ms: Optional response time in milliseconds - service_tier: Optional service tier the request was priced on - - Returns: - Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd - - Raises: - ValueError: If a model that is not a Model Router name is missing from the cost map - """ +def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float]: try: return generic_cost_per_token( model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier @@ -100,3 +75,38 @@ def cost_per_token( "Azure AI Model Router: model '%s' not in cost map, only the routing fee applies. Error: %s", model, e ) return 0.0, 0.0 + + +def cost_per_token( + model: str, + usage: Usage, + response_time_ms: float | None = 0.0, + request_model: str | None = None, + service_tier: str | None = None, +) -> tuple[float, float]: + """ + Price the response model's own tokens for Azure AI, plus the Model Router fee when the caller names the + router as the request model. + + completion_cost never passes request_model: it charges the fee once through + AzureModelRouterConfig.calculate_additional_costs as the "Azure Model Router Flat Cost" line of the cost + breakdown. A response priced as the router entry itself already carries the fee, so request_model adds + nothing on top of it, and a router deployment name that is missing from the cost map prices at zero here. + + Args: + model: str, the model name without provider prefix (from response) + usage: LiteLLM Usage block + response_time_ms: Optional response time in milliseconds + request_model: Optional[str], the original request model name; a Model Router name adds the routing fee + service_tier: Optional service tier the request was priced on + + Returns: + Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd + + Raises: + ValueError: If a model that is not a Model Router name is missing from the cost map + """ + prompt_cost, completion_cost = _response_model_cost(model=model, usage=usage, service_tier=service_tier) + if request_model is None or not _is_azure_model_router(request_model) or is_router_fee_entry(model): + return prompt_cost, completion_cost + return prompt_cost + calculate_azure_model_router_flat_cost(request_model, usage.prompt_tokens), completion_cost diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 20d0ec03a2a..0deb79d14d3 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -155,6 +155,39 @@ class TestAzureModelRouterFlatCost: with pytest.raises(Exception, match="no-such-azure-ai-model"): cost_per_token(model="no-such-azure-ai-model", usage=usage) + def test_request_model_through_the_router_adds_the_fee_once(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + prompt_cost, completion_cost_usd = cost_per_token( + model=ROUTED_MODEL, usage=ROUTED_USAGE, request_model="azure_ai/model-router" + ) + assert prompt_cost == pytest.approx(routed_prompt_cost + ROUTED_FEE, rel=1e-9) + assert completion_cost_usd == pytest.approx(routed_completion_cost, rel=1e-9) + + def test_request_model_that_is_not_the_router_adds_nothing(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + assert cost_per_token( + model=ROUTED_MODEL, usage=ROUTED_USAGE, request_model=f"azure_ai/{ROUTED_MODEL}" + ) == pytest.approx((routed_prompt_cost, routed_completion_cost), rel=1e-9) + + @pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) + def test_request_model_does_not_double_the_router_entry(self, router_entry_name: str) -> None: + prompt_cost, completion_cost_usd = cost_per_token( + model=router_entry_name, usage=ROUTED_USAGE, request_model=f"azure_ai/{router_entry_name}" + ) + assert prompt_cost == pytest.approx(ROUTED_FEE, rel=1e-9) + assert completion_cost_usd == 0.0 + + def test_public_cost_per_token_keeps_the_request_model_keyword(self) -> None: + routed_prompt_cost, routed_completion_cost = _routed_model_cost() + prompt_cost, completion_cost_usd = litellm.cost_per_token( + model=ROUTED_MODEL, + custom_llm_provider="azure_ai", + usage_object=ROUTED_USAGE, + request_model="azure_ai/model-router", + ) + assert prompt_cost == pytest.approx(routed_prompt_cost + ROUTED_FEE, rel=1e-9) + assert completion_cost_usd == pytest.approx(routed_completion_cost, rel=1e-9) + def test_flat_cost_helper(self) -> None: assert calculate_azure_model_router_flat_cost( model="azure-model-router", prompt_tokens=10_000 From 83d500bac9c375221120b4ae09e0e3cbd55f96b1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:29:26 -0700 Subject: [PATCH 084/411] fix(router): count deployment itpm reservations off the event loop --- litellm/litellm_core_utils/token_counter.py | 4 +-- .../io_token_rate_limit_check.py | 3 +- .../test_router/test_io_token_rate_limits.py | 28 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index e3a0d4c785a..e3af64fd3bf 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -315,12 +315,12 @@ def calculate_img_tokens( TokenCounterFunction = Callable[[str], int] - -EXTRAPOLATION_SAMPLES: Final = 16 """ Type for a function that counts tokens in a string. """ +EXTRAPOLATION_SAMPLES: Final = 16 + def _get_tiktoken_count_function( encode_length: Callable[[str], int], diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 01d42627001..dca5a2be845 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -21,6 +21,7 @@ import litellm from litellm import token_counter from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache +from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.router import RouterCacheEnum, RouterErrors from litellm.utils import get_utc_datetime @@ -466,7 +467,7 @@ async def async_io_token_pre_call_check( request_kwargs: Final = get_io_token_rate_limit_request_kwargs() _model: Final = (deployment.get("litellm_params") or {}).get("model") or "" - estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model) + estimated_input: Final = await asyncify(_estimate_input_tokens)(request_kwargs, model=_model) max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) dt: Final = get_utc_datetime() diff --git a/tests/test_litellm/test_router/test_io_token_rate_limits.py b/tests/test_litellm/test_router/test_io_token_rate_limits.py index a5a68271111..3cef1c7bb63 100644 --- a/tests/test_litellm/test_router/test_io_token_rate_limits.py +++ b/tests/test_litellm/test_router/test_io_token_rate_limits.py @@ -1039,3 +1039,31 @@ class TestContextSlotRetention: assert deployment is not None router._update_kwargs_with_deployment(deployment=deployment.model_dump(), kwargs=kwargs) assert get_io_token_rate_limit_request_kwargs() is kwargs + + +@pytest.mark.asyncio +async def test_the_deployment_itpm_reservation_counts_the_request_off_the_event_loop(): + from litellm.utils import get_utc_datetime + from tests.large_text import text + from tests.test_litellm.litellm_core_utils.event_loop_lag import ( + assert_loop_stayed_free, + timed_with_loop_lags, + warm_tokenizer, + ) + + dual_cache = DualCache() + check = ModelRateLimitingCheck(dual_cache=dual_cache) + warm_tokenizer("anthropic/claude-fable-5") + deployment = { + "litellm_params": {"model": "anthropic/claude-fable-5", "itpm": 10_000_000}, + "model_info": {"id": "io-loop-id"}, + "model_name": "claude", + } + set_io_token_rate_limit_request_kwargs({"messages": [{"role": "user", "content": text * 100}], "metadata": {}}) + + _, took, lags = await timed_with_loop_lags(lambda: check.async_pre_call_check(deployment)) + + minute = get_utc_datetime().strftime("%H-%M") + reserved = await dual_cache.async_get_cache(key=f"global_router:io-loop-id:anthropic/claude-fable-5:itpm:{minute}") + assert reserved > 100_000 + assert_loop_stayed_free(took, lags) From 55c10c1983c92dbad1d62dfc3c14aab94696639a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:34:34 -0700 Subject: [PATCH 085/411] fix(azure_ai): charge the router fee once for any router name and price grok-4-20 cache reads Direct litellm.cost_per_token callers that name a Model Router deployment as the model get the routing fee again, as they did before this branch, and the fee is still charged exactly once on every completion_cost path. The grok-4-20 entries bill cached prompt tokens at the input rate, since Azure has no cached-input meter for them, and the model_router twin carries the same limits and retirement date as model-router. The catalog test now exercises the cost calculator and map relations instead of pinning map fields. --- litellm/cost_calculator.py | 4 +- litellm/llms/azure_ai/cost_calculator.py | 33 ++- ...odel_prices_and_context_window_backup.json | 6 + model_prices_and_context_window.json | 6 + .../azure_ai/test_azure_ai_cost_calculator.py | 31 ++- ...azure_ai_foundry_catalog_model_metadata.py | 219 ++++++------------ 6 files changed, 125 insertions(+), 174 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e135503d11d..8a00ffa4d37 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -46,7 +46,7 @@ from litellm.llms.azure_ai.cost_calculator import ( cost_per_token as azure_ai_cost_per_token, ) from litellm.llms.azure_ai.cost_calculator import ( - is_router_fee_entry as azure_ai_is_router_fee_entry, + is_azure_model_router as azure_ai_is_model_router_name, ) from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.llms.bedrock.cost_calculation import ( @@ -1665,7 +1665,7 @@ def completion_cost( ) # Get additional costs from provider (e.g., routing fees, infrastructure costs) - if custom_llm_provider == "azure_ai" and not azure_ai_is_router_fee_entry(model): + if custom_llm_provider == "azure_ai" and not azure_ai_is_model_router_name(model): model_for_additional_costs = request_model_for_cost if completion_response is not None: hidden_params = getattr(completion_response, "_hidden_params", None) or {} diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index e57ba055587..5934525eca3 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -11,7 +11,7 @@ from litellm.types.utils import Usage from litellm.utils import get_model_info -def _is_azure_model_router(model: str) -> bool: +def is_azure_model_router(model: str) -> bool: """ Check if the model is Azure AI Foundry Model Router. @@ -54,7 +54,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl Returns: float: The flat cost in USD, or 0.0 if not applicable """ - if not _is_azure_model_router(model): + if not is_azure_model_router(model): return 0.0 model_info: Final = get_model_info(model=_router_fee_entry_name(model), custom_llm_provider="azure_ai") router_flat_cost_per_token: Final = model_info.get("input_cost_per_token", 0) @@ -69,7 +69,7 @@ def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier ) except Exception as e: - if not _is_azure_model_router(model): + if not is_azure_model_router(model): raise verbose_logger.debug( "Azure AI Model Router: model '%s' not in cost map, only the routing fee applies. Error: %s", model, e @@ -77,6 +77,16 @@ def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> return 0.0, 0.0 +def _router_fee_name(model: str, request_model: str | None) -> str | None: + if is_router_fee_entry(model): + return None + if is_azure_model_router(model): + return model + if request_model is not None and is_azure_model_router(request_model): + return request_model + return None + + def cost_per_token( model: str, usage: Usage, @@ -85,13 +95,15 @@ def cost_per_token( service_tier: str | None = None, ) -> tuple[float, float]: """ - Price the response model's own tokens for Azure AI, plus the Model Router fee when the caller names the - router as the request model. + Price the response model's own tokens for Azure AI, plus the Model Router fee exactly once when either the + priced name or request_model is a Model Router name. - completion_cost never passes request_model: it charges the fee once through + A response priced as the router entry itself already carries the fee, so nothing is added on top of it. A + router deployment name that is missing from the cost map prices at the fee alone. + + completion_cost passes only the priced name: when that name is a routed model it adds the fee itself through AzureModelRouterConfig.calculate_additional_costs as the "Azure Model Router Flat Cost" line of the cost - breakdown. A response priced as the router entry itself already carries the fee, so request_model adds - nothing on top of it, and a router deployment name that is missing from the cost map prices at zero here. + breakdown, and when the name is router-shaped the fee is already in the prompt cost returned here. Args: model: str, the model name without provider prefix (from response) @@ -107,6 +119,7 @@ def cost_per_token( ValueError: If a model that is not a Model Router name is missing from the cost map """ prompt_cost, completion_cost = _response_model_cost(model=model, usage=usage, service_tier=service_tier) - if request_model is None or not _is_azure_model_router(request_model) or is_router_fee_entry(model): + fee_name: Final = _router_fee_name(model=model, request_model=request_model) + if fee_name is None: return prompt_cost, completion_cost - return prompt_cost + calculate_azure_model_router_flat_cost(request_model, usage.prompt_tokens), completion_cost + return prompt_cost + calculate_azure_model_router_flat_cost(fee_name, usage.prompt_tokens), completion_cost diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 483ebd2431c..674a8b98304 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4060,9 +4060,13 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/model_router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -10754,6 +10758,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", @@ -10771,6 +10776,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 483ebd2431c..674a8b98304 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4060,9 +4060,13 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/model_router": { + "deprecation_date": "2027-05-20", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -10754,6 +10758,7 @@ "supports_web_search": true }, "azure_ai/grok-4-20-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", @@ -10771,6 +10776,7 @@ "supports_reasoning": true }, "azure_ai/grok-4-20-non-reasoning": { + "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 0deb79d14d3..7df14b91741 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -11,9 +11,9 @@ import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.cost_calculator import ( - _is_azure_model_router, calculate_azure_model_router_flat_cost, cost_per_token, + is_azure_model_router, ) from litellm.types.utils import Choices, Message, ModelResponse, Usage from litellm.utils import get_model_info @@ -54,7 +54,7 @@ class TestAzureModelRouterDetection: ) def test_is_azure_model_router(self, model: str, expected: bool): """Test Azure Model Router detection.""" - assert _is_azure_model_router(model) == expected + assert is_azure_model_router(model) == expected class TestAzureModelRouterPrefix: @@ -130,11 +130,21 @@ def _routed_model_cost() -> tuple[float, float]: @pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterFlatCost: - """cost_per_token prices the response model only; the router fee is the cost breakdown's own line item.""" + """cost_per_token charges the router fee once, for whichever router name the caller gives it.""" - def test_unmapped_router_deployment_name_prices_at_zero(self) -> None: + def test_unmapped_router_deployment_name_prices_the_fee(self) -> None: usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - assert cost_per_token(model="azure-model-router", usage=usage) == (0.0, 0.0) + prompt_cost, completion_cost_usd = cost_per_token(model="azure-model-router", usage=usage) + assert prompt_cost == pytest.approx(1000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) + assert completion_cost_usd == 0.0 + + def test_router_deployment_name_as_both_names_charges_the_fee_once(self) -> None: + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + prompt_cost, completion_cost_usd = cost_per_token( + model="model_router/my-deployment", usage=usage, request_model="azure_ai/model_router/my-deployment" + ) + assert prompt_cost == pytest.approx(1000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) + assert completion_cost_usd == 0.0 @pytest.mark.parametrize("router_entry_name", ["model_router", "model-router"]) def test_router_entry_prices_its_own_fee(self, router_entry_name: str) -> None: @@ -209,7 +219,8 @@ class TestAzureModelRouterFlatCost: @pytest.mark.usefixtures("local_model_cost_map") class TestAzureModelRouterCostBreakdown: - """completion_cost charges the router fee exactly once, as the cost breakdown's additional cost line.""" + """completion_cost charges the router fee exactly once: as the breakdown's additional cost line when a routed + model is priced as itself, inside the input cost when the priced name is the router.""" def test_unmapped_router_deployment_name_costs_only_the_fee(self) -> None: cost = completion_cost( @@ -219,7 +230,7 @@ class TestAzureModelRouterCostBreakdown: ) assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) - def test_fee_is_the_breakdown_line_item_for_an_unmapped_router_name(self) -> None: + def test_unmapped_router_name_carries_the_fee_as_its_input_cost(self) -> None: logging_obj = _router_logging("azure-model-router") cost = completion_cost( completion_response=_azure_ai_response("azure-model-router"), @@ -229,10 +240,8 @@ class TestAzureModelRouterCostBreakdown: ) breakdown = logging_obj.cost_breakdown assert breakdown is not None - assert breakdown["input_cost"] == 0.0 - assert breakdown.get("additional_costs") == pytest.approx( - {"Azure Model Router Flat Cost": ROUTED_FEE}, rel=1e-9 - ) + assert breakdown["input_cost"] == pytest.approx(ROUTED_FEE, rel=1e-9) + assert "additional_costs" not in breakdown assert cost == pytest.approx(ROUTED_FEE, rel=1e-9) def test_router_request_with_routed_response_charges_the_fee_once(self) -> None: diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index fab9be1b42c..19b082edd8a 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -5,131 +5,34 @@ from typing import Final import pytest from pydantic import TypeAdapter -from litellm import cost_per_token, get_model_info +from litellm import completion_cost, cost_per_token from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.types.utils import TranscriptionResponse REPO_ROOT: Final = Path(__file__).parents[4] +MAIN_COST_MAP: Final = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_COST_MAP: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) -AZURE_OPENAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" -FOUNDRY_AOAI_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/" -FOUNDRY_COHERE_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/" -FOUNDRY_GROK_PRICING: Final = "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/" +AZURE_PRICING_PREFIX: Final = "https://azure.microsoft.com/en-us/pricing/details/" +A_MILLION: Final = 1_000_000 @dataclass(frozen=True, slots=True) class TokenPricedCatalogModel: catalog_name: str - mode: str - source: str - input_cost_per_token: float - output_cost_per_token: float - max_input_tokens: int - max_output_tokens: int - cache_read_input_token_cost: float | None - deprecation_date: str | None - supported_flags: tuple[str, ...] + dollars_per_million_input: float + dollars_per_million_output: float TOKEN_PRICED_MODELS: Final = ( - TokenPricedCatalogModel( - catalog_name="gpt-chat-latest", - mode="chat", - source=AZURE_OPENAI_PRICING, - input_cost_per_token=5e-06, - output_cost_per_token=3e-05, - max_input_tokens=272000, - max_output_tokens=128000, - cache_read_input_token_cost=5e-07, - deprecation_date="2026-12-02", - supported_flags=( - "supports_function_calling", - "supports_prompt_caching", - "supports_reasoning", - "supports_response_schema", - "supports_tool_choice", - "supports_vision", - "supports_web_search", - ), - ), - TokenPricedCatalogModel( - catalog_name="codex-mini", - mode="responses", - source=AZURE_OPENAI_PRICING, - input_cost_per_token=1.5e-06, - output_cost_per_token=6e-06, - max_input_tokens=200000, - max_output_tokens=100000, - cache_read_input_token_cost=3.75e-07, - deprecation_date="2026-11-15", - supported_flags=( - "supports_function_calling", - "supports_prompt_caching", - "supports_reasoning", - "supports_vision", - ), - ), - TokenPricedCatalogModel( - catalog_name="model-router", - mode="chat", - source=FOUNDRY_AOAI_PRICING, - input_cost_per_token=1.4e-07, - output_cost_per_token=0.0, - max_input_tokens=200000, - max_output_tokens=32768, - cache_read_input_token_cost=None, - deprecation_date="2027-05-20", - supported_flags=(), - ), - TokenPricedCatalogModel( - catalog_name="cohere-command-a", - mode="chat", - source=FOUNDRY_COHERE_PRICING, - input_cost_per_token=2.5e-06, - output_cost_per_token=1e-05, - max_input_tokens=131072, - max_output_tokens=8182, - cache_read_input_token_cost=None, - deprecation_date=None, - supported_flags=("supports_function_calling", "supports_tool_choice"), - ), - TokenPricedCatalogModel( - catalog_name="grok-4-20-reasoning", - mode="chat", - source=FOUNDRY_GROK_PRICING, - input_cost_per_token=1.25e-06, - output_cost_per_token=2.5e-06, - max_input_tokens=262000, - max_output_tokens=8192, - cache_read_input_token_cost=None, - deprecation_date="2027-04-06", - supported_flags=( - "supports_function_calling", - "supports_reasoning", - "supports_response_schema", - "supports_tool_choice", - "supports_vision", - "supports_web_search", - ), - ), - TokenPricedCatalogModel( - catalog_name="grok-4-20-non-reasoning", - mode="chat", - source=FOUNDRY_GROK_PRICING, - input_cost_per_token=1.25e-06, - output_cost_per_token=2.5e-06, - max_input_tokens=262000, - max_output_tokens=8192, - cache_read_input_token_cost=None, - deprecation_date="2027-04-06", - supported_flags=( - "supports_function_calling", - "supports_response_schema", - "supports_tool_choice", - "supports_vision", - "supports_web_search", - ), - ), + TokenPricedCatalogModel("gpt-chat-latest", 5.0, 30.0), + TokenPricedCatalogModel("codex-mini", 1.5, 6.0), + TokenPricedCatalogModel("model-router", 0.14, 0.0), + TokenPricedCatalogModel("cohere-command-a", 2.5, 10.0), + TokenPricedCatalogModel("grok-4-20-reasoning", 1.25, 2.5), + TokenPricedCatalogModel("grok-4-20-non-reasoning", 1.25, 2.5), ) +GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning") CATALOG_NAMES: Final = tuple(spec.catalog_name for spec in TOKEN_PRICED_MODELS) + ("whisper",) @@ -137,60 +40,74 @@ def _cost_map_entry(path: Path, catalog_name: str) -> dict[str, object]: return COST_MAP_ADAPTER.validate_json(path.read_bytes())[f"azure_ai/{catalog_name}"] +@pytest.mark.parametrize("catalog_name", CATALOG_NAMES) +def test_azure_ai_catalog_name_routes_to_azure_ai(catalog_name: str) -> None: + routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{catalog_name}") + assert (routed_model, provider) == (catalog_name, "azure_ai") + + @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_name_is_priced_and_routed(spec: TokenPricedCatalogModel) -> None: - routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{spec.catalog_name}") - assert (routed_model, provider) == (spec.catalog_name, "azure_ai") - - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["litellm_provider"] == "azure_ai" - assert info["mode"] == spec.mode - assert info["input_cost_per_token"] == spec.input_cost_per_token - assert info["output_cost_per_token"] == spec.output_cost_per_token - assert info["cache_read_input_token_cost"] == spec.cache_read_input_token_cost - assert info["max_input_tokens"] == spec.max_input_tokens - assert info["max_output_tokens"] == spec.max_output_tokens - assert info["max_tokens"] == spec.max_output_tokens - for flag in spec.supported_flags: - assert info[flag] is True, flag +def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: + prompt_cost, completion_cost_usd = cost_per_token( + model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=A_MILLION + ) + assert prompt_cost == pytest.approx(spec.dollars_per_million_input) + assert completion_cost_usd == pytest.approx(spec.dollars_per_million_output) @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize( - "spec", - [spec for spec in TOKEN_PRICED_MODELS if spec.catalog_name != "model-router"], - ids=lambda spec: spec.catalog_name, -) -def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: - prompt_cost, completion_cost = cost_per_token( - model=f"azure_ai/{spec.catalog_name}", prompt_tokens=1_000_000, completion_tokens=1_000_000 +@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) +def test_azure_ai_catalog_name_prices_the_same_in_any_casing(spec: TokenPricedCatalogModel) -> None: + lowercase_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) + upper_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name.upper()}", prompt_tokens=A_MILLION, completion_tokens=0) + assert upper_cost == lowercase_cost + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES) +def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None: + uncached_prompt_cost, _ = cost_per_token(model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) + cached_prompt_cost, _ = cost_per_token( + model=f"azure_ai/{catalog_name}", + prompt_tokens=A_MILLION, + completion_tokens=0, + cache_read_input_tokens=A_MILLION, ) - assert prompt_cost == pytest.approx(spec.input_cost_per_token * 1_000_000) - assert completion_cost == pytest.approx(spec.output_cost_per_token * 1_000_000) + assert uncached_prompt_cost > 0 + assert cached_prompt_cost == pytest.approx(uncached_prompt_cost) @pytest.mark.usefixtures("local_model_cost_map") def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None: - routed_model, provider, _, _ = get_llm_provider(model="azure_ai/whisper") - assert (routed_model, provider) == ("whisper", "azure_ai") - - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["mode"] == "audio_transcription" - assert info["input_cost_per_second"] == 0.0001 - assert info["output_cost_per_second"] == 0.0001 + transcription: Final = TranscriptionResponse(text="hello") + transcription._hidden_params = { # pyright: ignore[reportPrivateUsage] # TranscriptionResponse exposes no public hidden-params setter + "custom_llm_provider": "azure_ai", + "model": "azure_ai/whisper", + "audio_transcription_duration": 3600, + } + cost = completion_cost( + completion_response=transcription, + model="azure_ai/whisper", + custom_llm_provider="azure_ai", + call_type="atranscription", + ) + assert cost == pytest.approx(0.36) @pytest.mark.parametrize("catalog_name", CATALOG_NAMES) def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> None: - main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", catalog_name) - backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json", catalog_name) + main_entry = _cost_map_entry(MAIN_COST_MAP, catalog_name) + backup_entry = _cost_map_entry(BACKUP_COST_MAP, catalog_name) - assert str(main_entry["source"]).startswith("https://azure.microsoft.com/en-us/pricing/details/") + assert str(main_entry["source"]).startswith(AZURE_PRICING_PREFIX) assert backup_entry == main_entry -@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_entry_carries_its_retirement_date(spec: TokenPricedCatalogModel) -> None: - entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json", spec.catalog_name) - assert entry.get("deprecation_date") == spec.deprecation_date +def test_azure_ai_model_router_spellings_share_one_entry() -> None: + underscore_entry = _cost_map_entry(MAIN_COST_MAP, "model_router") + hyphen_entry = _cost_map_entry(MAIN_COST_MAP, "model-router") + + assert {k: v for k, v in underscore_entry.items() if k != "comment"} == { + k: v for k, v in hyphen_entry.items() if k != "comment" + } From 1de369a1a446b7ecc1ed60a0c04d26bed1601c3a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:54:37 -0700 Subject: [PATCH 086/411] fix(passthrough): bound azure relays to the key's model group and reject foreign deployment segments --- litellm/proxy/auth/auth_utils.py | 10 ++ .../llm_passthrough_endpoints.py | 41 ++++++ .../test_azure_passthrough_transformation.py | 4 +- .../proxy/auth/test_auth_utils.py | 32 +++++ .../test_llm_pass_through_endpoints.py | 134 +++++++++++++++++- tests/test_litellm/test_router.py | 4 +- 6 files changed, 217 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 1e4836654a1..dd7df2b3fd9 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1985,9 +1985,19 @@ def get_model_from_request( bedrock_model: Final = _model_from_bedrock_route(route) return model if bedrock_model is None else bedrock_model + if route.lower().startswith(("/azure/", "/azure_ai/")): + azure_model: Final = _router_model_from_azure_route(route, llm_router) + return model if azure_model is None else azure_model + return model +def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str | None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import azure_router_model_in_endpoint + + return azure_router_model_in_endpoint(re.sub(r"^/azure(?:_ai)?/", "", route, flags=re.IGNORECASE), llm_router) + + def _model_from_bedrock_route(route: str) -> str | None: from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _extract_model_from_bedrock_endpoint, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 839c9d47c6f..47b3c56e66a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -78,6 +78,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, ) from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials +from litellm.types.router import LiteLLMParamsTypedDict from litellm.types.utils import LlmProviders from litellm.types.vector_stores import LiteLLM_ManagedVectorStore from litellm.utils import ProviderConfigManager @@ -120,6 +121,37 @@ def is_passthrough_request_using_router_model(request_body: dict, llm_router: li return False +def azure_router_model_in_endpoint(endpoint: str, llm_router: litellm.Router | None) -> str | None: + parts: Final = endpoint.split("/") + if len(parts) < 2: + return None + return next((part for part in parts if is_known_model(part, llm_router)), None) + + +AZURE_DEPLOYMENT_SEGMENT: Final = re.compile(r"(? str: + model: Final = litellm_params.get("model", "") + try: + return get_llm_provider(model=model, custom_llm_provider=litellm_params.get("custom_llm_provider"))[0] + except litellm.BadRequestError: + return model + + +def foreign_azure_deployment(endpoint: str, model_group: str, llm_router: litellm.Router) -> str | None: + match: Final = AZURE_DEPLOYMENT_SEGMENT.search(endpoint) + if match is None: + return None + deployment: Final = match.group(1) + if deployment == model_group: + return None + served: Final = frozenset( + _deployment_model_name(row["litellm_params"]) for row in llm_router.get_model_list(model_name=model_group) or () + ) + return None if deployment in served else deployment + + def is_passthrough_request_streaming(request_body: object) -> bool: """ Returns True if the request is streaming. @@ -1523,6 +1555,15 @@ async def _relay_azure_router_model( is_streaming_request: bool, user_api_key_dict: UserAPIKeyAuth, ) -> Response: + foreign_deployment: Final = foreign_azure_deployment(endpoint, model, llm_router) + if foreign_deployment is not None: + raise HTTPException( + status_code=400, + detail={ + "error": f"deployment '{foreign_deployment}' in the path is not served by model group '{model}'; " + "put the model group name in the deployments segment" + }, + ) try: result: Final = await llm_router.allm_passthrough_route( model=model, diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 2723a13c3fc..54e27b34a59 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -334,12 +334,12 @@ def test_azure_passthrough_url_rewrites_the_model_group_only_as_a_whole_segment( api_base="https://my-resource.openai.azure.com", api_key="key", model="gpt-4.1-mini", - endpoint="gpt/openai/deployments/gpt-4o/chat/completions", + endpoint="gpt/openai/deployments/gpt-4.1-mini/chat/completions", request_query_params={"api-version": "2024-10-21"}, litellm_params={"litellm_metadata": {"model_group": "gpt"}}, ) - assert str(url) == "https://my-resource.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21" + assert str(url) == "https://my-resource.openai.azure.com/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2024-10-21" @pytest.mark.parametrize( diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index a996de4d40c..ccb707a8e52 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -528,6 +528,38 @@ def test_get_model_from_request_bedrock_unparseable_endpoint_keeps_body_model(): ) +def _azure_relay_router(): + from litellm.router import Router + + return Router( + model_list=[ + { + "model_name": "gpt", + "litellm_params": {"model": "azure_ai/gpt-5.4-mini", "api_base": "https://a.services.ai.azure.com", "api_key": "k"}, + }, + { + "model_name": "other-group", + "litellm_params": {"model": "azure/gpt-5.4", "api_base": "https://b.openai.azure.com", "api_key": "k"}, + }, + ] + ) + + +@pytest.mark.parametrize( + "route, request_data, expected", + [ + ("/azure_ai/other-group/openai/deployments/other-group/chat/completions", {"model": "gpt"}, "other-group"), + ("/azure_ai/other-group/models/chat/completions", {}, "other-group"), + ("/azure/openai/deployments/gpt/chat/completions", {"model": "other-group"}, "gpt"), + ("/azure/openai/deployments/gpt/chat/completions", {}, "gpt"), + ("/azure/openai/deployments/my-azure-deployment/chat/completions", {"model": "gpt"}, "gpt"), + ("/azure_ai/gpt", {"model": "other-group"}, "other-group"), + ], +) +def test_get_model_from_request_azure_relay_routes_use_the_model_group_in_the_path(route, request_data, expected): + assert get_model_from_request(request_data=request_data, route=route, llm_router=_azure_relay_router()) == expected + + def test_get_model_from_request_includes_file_endpoint_header_model(): assert ( get_model_from_request( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 6ff14157468..fe4400df704 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -4999,7 +4999,11 @@ class TestPassthroughRouterModelBudgetReservation: monkeypatch.setattr(proxy_server, "llm_router", RecordingRouter()) monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) - monkeypatch.setattr(ep, "is_passthrough_request_using_router_model", lambda *a, **k: True) + monkeypatch.setattr( + ep, + "is_passthrough_request_using_router_model", + lambda request_body, llm_router=None: request_body.get("model") in ("gpt-5", "router-model"), + ) return captured def _assert_metadata_carries_attribution(self, captured: list[dict], user_api_key_dict: UserAPIKeyAuth) -> None: @@ -5098,7 +5102,11 @@ class TestAzureRouterModelStreamingDispatch: monkeypatch.setattr(proxy_server, "llm_router", StreamingRouter()) monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) - monkeypatch.setattr(ep, "is_passthrough_request_using_router_model", lambda *a, **k: True) + monkeypatch.setattr( + ep, + "is_passthrough_request_using_router_model", + lambda request_body, llm_router=None: request_body.get("model") in ("gpt-5", "router-model"), + ) request = MagicMock(spec=Request) request.method = "POST" @@ -5158,7 +5166,11 @@ class TestAzureRouterModelStreamingKeepalive: monkeypatch.setattr(proxy_server, "llm_router", StreamingRouter()) monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) - monkeypatch.setattr(ep, "is_passthrough_request_using_router_model", lambda *a, **k: True) + monkeypatch.setattr( + ep, + "is_passthrough_request_using_router_model", + lambda request_body, llm_router=None: request_body.get("model") in ("gpt-5", "router-model"), + ) request = MagicMock(spec=Request) request.method = "POST" @@ -5225,7 +5237,11 @@ class TestRouterModelRelayUpstreamContract: monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) - monkeypatch.setattr(ep, "is_passthrough_request_using_router_model", lambda *a, **k: True) + monkeypatch.setattr( + ep, + "is_passthrough_request_using_router_model", + lambda request_body, llm_router=None: request_body.get("model") in ("gpt-5", "router-model"), + ) def _recording_router(self, captured: list[dict]): class RecordingRouter: @@ -5327,3 +5343,113 @@ async def test_bedrock_count_tokens_error_forwards_provider_headers(): assert exc_info.value.status_code == 500 assert exc_info.value.headers["llm_provider-x-amzn-requestid"] == "req-count-tokens-500" + + +class _AzureGroupRouter: + def __init__(self, captured: list[dict]) -> None: + self.captured = captured + + def get_model_names(self, team_id=None): + return ["gpt", "other-group"] + + def get_model_list(self, model_name=None, team_id=None): + rows = [ + {"model_name": "gpt", "litellm_params": {"model": "azure_ai/gpt-5.4-mini", "api_key": "k"}}, + {"model_name": "other-group", "litellm_params": {"model": "azure/gpt-5.4", "api_key": "k"}}, + ] + return [row for row in rows if model_name is None or row["model_name"] == model_name] + + async def allm_passthrough_route(self, **kwargs): + self.captured.append(kwargs) + return httpx.Response(200, json={"ok": True}) + + +class TestAzureRelayDeploymentSegment: + """A key allowed one model group must not reach another deployment by naming it in the + ``openai/deployments/`` segment while the group segment picks the credential.""" + + @pytest.mark.parametrize( + "endpoint, expected", + [ + ("gpt/openai/deployments/gpt/chat/completions", None), + ("openai/deployments/gpt/chat/completions", None), + ("gpt/openai/deployments/gpt-5.4-mini/chat/completions", None), + ("gpt/models/chat/completions", None), + ("gpt/openai/deployments/gpt-5.4/chat/completions", "gpt-5.4"), + ("gpt/openai/deployments/other-group/chat/completions", "other-group"), + ("openai/deployments/victim/gpt/chat/completions", "victim"), + ], + ) + def test_foreign_azure_deployment_names_a_segment_outside_the_group(self, endpoint, expected): + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import foreign_azure_deployment + + assert foreign_azure_deployment(endpoint, "gpt", _AzureGroupRouter([])) == expected + + @pytest.mark.parametrize( + "endpoint, expected", + [ + ("other-group/openai/deployments/other-group/chat/completions", "other-group"), + ("openai/deployments/gpt/chat/completions", "gpt"), + ("openai/deployments/my-azure-deployment/chat/completions", None), + ("gpt", None), + ], + ) + def test_azure_router_model_in_endpoint_matches_the_relay_decision(self, endpoint, expected): + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import azure_router_model_in_endpoint + + assert azure_router_model_in_endpoint(endpoint, _AzureGroupRouter([])) == expected + + def _install(self, monkeypatch, body: dict) -> list[dict]: + import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep + import litellm.proxy.proxy_server as proxy_server + + captured: list[dict] = [] + + async def fake_get_request_body(_request): + return body + + monkeypatch.setattr(proxy_server, "llm_router", _AzureGroupRouter(captured)) + monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) + return captured + + def _request(self) -> Request: + request = MagicMock(spec=Request) + request.method = "POST" + request.headers = {"content-type": "application/json"} + request.query_params = {} + return request + + @pytest.mark.asyncio + async def test_azure_relay_rejects_a_deployment_the_group_does_not_serve(self, monkeypatch): + from fastapi import HTTPException + + captured = self._install(monkeypatch, {"model": "gpt", "messages": []}) + + with pytest.raises(HTTPException) as exc_info: + await azure_proxy_route( + endpoint="gpt/openai/deployments/gpt-5.4/chat/completions", + request=self._request(), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token", models=["gpt"]), + ) + + assert exc_info.value.status_code == 400 + assert "gpt-5.4" in exc_info.value.detail["error"] + assert captured == [] + + @pytest.mark.asyncio + async def test_azure_relay_dispatches_the_group_and_its_own_deployment_name(self, monkeypatch): + captured = self._install(monkeypatch, {"model": "gpt", "messages": []}) + + for endpoint in ( + "gpt/openai/deployments/gpt/chat/completions", + "gpt/openai/deployments/gpt-5.4-mini/chat/completions", + ): + await azure_proxy_route( + endpoint=endpoint, + request=self._request(), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token", models=["gpt"]), + ) + + assert [call["model"] for call in captured] == ["gpt", "gpt"] diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 204c6f109ff..a766f0c9db1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5053,8 +5053,8 @@ def test_get_deployment_model_info_base_model_merge_priority(): ( "gpt", {"model": "azure_ai/gpt-5.4-mini", "api_base": "https://my-resource.services.ai.azure.com", "api_key": "key"}, - "gpt/openai/deployments/gpt-4o/chat/completions", - "gpt-5.4-mini/openai/deployments/gpt-4o/chat/completions", + "gpt/openai/deployments/gpt-5.4-mini/chat/completions", + "gpt-5.4-mini/openai/deployments/gpt-5.4-mini/chat/completions", ), ( "aws/anthropic/bedrock-claude", From 52e9cfe254355b2ffdb75fe314c2b62b467f5018 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:57:00 -0700 Subject: [PATCH 087/411] fix(azure): price streamed Responses relays and keep a full-URL api_base from doubling the native path --- litellm/litellm_core_utils/litellm_logging.py | 5 +- .../llms/azure/passthrough/transformation.py | 23 +++++++- .../azure_ai/passthrough/transformation.py | 5 +- .../base_llm/passthrough/transformation.py | 2 +- .../test_azure_passthrough_transformation.py | 47 +++++++++++++++ ...est_azure_ai_passthrough_transformation.py | 57 +++++++++++++++++++ 6 files changed, 129 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ca2cca5360f..ec988b200d0 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -119,7 +119,6 @@ from litellm.types.utils import ( CachingDetails, CallTypes, CostBreakdown, - CostResponseTypes, CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, EmbeddingResponse, @@ -201,7 +200,7 @@ if TYPE_CHECKING: from mcp.types import EmbeddedResource, ImageContent, TextContent from litellm.integrations.otel.logger import OpenTelemetryV2 - from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, LoggedRelayResponse try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( EnterpriseCallbackControls, @@ -2363,7 +2362,7 @@ class Logging(LiteLLMLoggingBaseClass): self, raw_bytes: list[bytes], provider_config: "BasePassthroughConfig", - ) -> Optional["CostResponseTypes"]: + ) -> Optional["LoggedRelayResponse"]: all_chunks: Final = provider_config._convert_raw_bytes_to_str_lines(raw_bytes) complete_streaming_response: Final = provider_config.handle_logging_collected_chunks( all_chunks=all_chunks, diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index bdb854bb87a..ab7fe412628 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -23,7 +23,6 @@ if TYPE_CHECKING: from httpx import URL from litellm.llms.base_llm.passthrough.transformation import LoggedRelayResponse - from litellm.types.utils import CostResponseTypes class RelayedChatRequest(BaseModel): @@ -42,13 +41,29 @@ def _relayed_messages(litellm_logging_obj: Logging) -> Sequence[Mapping[str, obj return details.request_data.messages if details.request_data else None +RESPONSES_RELAY_SHAPE: Final = RelayShape("/responses", CallTypes.aresponses, ResponsesAPIResponse.model_validate) + OPENAI_RELAY_SHAPES: Final = ( RelayShape("/embeddings", CallTypes.aembedding, EmbeddingResponse.model_validate), - RelayShape("/responses", CallTypes.aresponses, ResponsesAPIResponse.model_validate), + RESPONSES_RELAY_SHAPE, RelayShape("/images/generations", CallTypes.aimage_generation, ImageResponse.model_validate), ) +def logged_responses_stream(all_chunks: Sequence[str], logging_obj: Logging) -> ResponsesAPIResponse | None: + from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + + terminal_response: Final = OpenAIResponsesAPIConfig.parse_terminal_response_from_stream_chunks( + all_chunks=list(all_chunks) + ) + if terminal_response is None: + return None + logging_obj.call_type = ( + RESPONSES_RELAY_SHAPE.call_type.value + ) # rebind-ok: routes cost calculation to the relayed shape's pricing path + return terminal_response + + class AzurePassthroughConfig(BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: return bool(request_data.get("stream")) @@ -157,11 +172,13 @@ class AzurePassthroughConfig(BasePassthroughConfig): model: str, custom_llm_provider: str, endpoint: str, - ) -> Optional["CostResponseTypes"]: + ) -> Optional["LoggedRelayResponse"]: from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import ( OpenAIPassthroughLoggingHandler, ) + if f"/{endpoint.strip('/')}".endswith(RESPONSES_RELAY_SHAPE.path_suffix): + return logged_responses_stream(all_chunks, litellm_logging_obj) if "chat/completions" not in endpoint: return None diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index e854874eaee..1689755b042 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -30,7 +30,6 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.llms.base_llm.passthrough.transformation import LoggedRelayResponse - from litellm.types.utils import CostResponseTypes EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({}) @@ -111,8 +110,8 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): if base_target_url is None: raise ValueError("Azure AI api base not found: set `api_base` on the deployment or AZURE_AI_API_BASE") - root: Final = foundry_root(base_target_url) native_endpoint: Final = strip_leading_model_segment(endpoint, (model, model_group_from(litellm_params))) + root: Final = foundry_root(base_target_url).removesuffix(f"/{native_endpoint.strip('/')}") query_params: Final = relay_query_params( request_query_params, api_version_from(litellm_params), base_target_url ) @@ -200,7 +199,7 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): model: str, custom_llm_provider: str, endpoint: str, - ) -> CostResponseTypes | None: + ) -> LoggedRelayResponse | None: from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig return AzurePassthroughConfig().handle_logging_collected_chunks( diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index 93ec5a09e62..dc857dfc808 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -179,7 +179,7 @@ class BasePassthroughConfig(BaseLLMModelInfo): model: str, custom_llm_provider: str, endpoint: str, - ) -> CostResponseTypes | None: + ) -> LoggedRelayResponse | None: return None def _convert_raw_bytes_to_str_lines(self, raw_bytes: list[bytes]) -> list[str]: diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 54e27b34a59..42dd65ea661 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -283,6 +283,53 @@ def test_azure_passthrough_streaming_chunks_for_unknown_endpoint_return_none(): assert response is None +def _azure_responses_stream_chunks(terminal_event: str | None = "response.completed") -> list[str]: + in_progress = {**RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None} + events = [ + ("response.created", {"type": "response.created", "sequence_number": 0, "response": in_progress}), + ( + "response.output_text.delta", + {"type": "response.output_text.delta", "sequence_number": 1, "item_id": "msg_1", "delta": "hi"}, + ), + ] + ([(terminal_event, {"type": terminal_event, "sequence_number": 2, "response": RESPONSES_BODY})] if terminal_event else []) + return [line for name, payload in events for line in (f"event: {name}", _sse_line(payload))] + + +def test_azure_passthrough_streaming_responses_chunks_are_costed_per_token(): + logging_obj = _relay_logging_obj("gpt-4.1-mini") + + response = AzurePassthroughConfig().handle_logging_collected_chunks( + all_chunks=_azure_responses_stream_chunks(), + litellm_logging_obj=logging_obj, + model="gpt-4.1-mini", + custom_llm_provider="azure", + endpoint="openai/responses", + ) + info = litellm.get_model_info("azure/gpt-4.1-mini") + + assert isinstance(response, ResponsesAPIResponse) + assert response.usage.input_tokens == 1000 + assert logging_obj.call_type == "aresponses" + assert logging_obj._response_cost_calculator(result=response) == pytest.approx( + 1000 * info["input_cost_per_token"] + 100 * info["output_cost_per_token"] + ) + + +def test_azure_passthrough_streaming_responses_without_a_terminal_event_are_not_costed(): + logging_obj = _relay_logging_obj("gpt-4.1-mini") + + response = AzurePassthroughConfig().handle_logging_collected_chunks( + all_chunks=_azure_responses_stream_chunks(terminal_event=None), + litellm_logging_obj=logging_obj, + model="gpt-4.1-mini", + custom_llm_provider="azure", + endpoint="openai/responses", + ) + + assert response is None + assert logging_obj.call_type == "allm_passthrough_route" + + def _complete_url(request_query_params: dict, litellm_params: dict) -> httpx.URL: url, _ = AzurePassthroughConfig().get_complete_url( api_base="https://my-resource.openai.azure.com", diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index 8b6c76952d6..0b23da984e8 100644 --- a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -86,6 +86,22 @@ def test_api_base_that_already_ends_in_models_is_cut_back_to_the_foundry_root(): assert base == FOUNDRY_BASE +def test_full_url_api_base_that_already_ends_with_the_native_path_is_not_doubled(): + model_router_url = "https://my-resource.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions" + + url, base = AzureAIPassthroughConfig().get_complete_url( + api_base=f"{model_router_url}?api-version=2025-01-01-preview", + api_key="key", + model="model_router/model-router", + endpoint="model-router/chat/completions", + request_query_params=None, + litellm_params={"litellm_metadata": {"model_group": "model-router"}}, + ) + + assert str(url) == f"{model_router_url}?api-version=2025-01-01-preview" + assert base == "https://my-resource.cognitiveservices.azure.com/openai/deployments/model-router" + + def test_parse_relay_under_a_models_api_base_targets_the_foundry_root(): url, _ = AzureAIPassthroughConfig().get_complete_url( api_base=f"{FOUNDRY_BASE}/models", @@ -490,3 +506,44 @@ def test_streaming_chat_completion_chunks_are_costed_like_azure(): assert isinstance(response, ModelResponse) assert response.choices[0].message.content == "hi" assert response.usage.total_tokens == 4 + + +def test_streaming_responses_chunks_through_a_router_relay_are_costed_like_azure(): + completed = { + "type": "response.completed", + "sequence_number": 2, + "response": { + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.4-mini", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + "usage": {"input_tokens": 1000, "output_tokens": 100, "total_tokens": 1100}, + }, + } + logging_obj = _relay_logging_obj("gpt-5.4-mini", FOUNDRY_BASE) + + response = AzureAIPassthroughConfig().handle_logging_collected_chunks( + all_chunks=["event: response.completed", "data: " + json.dumps(completed)], + litellm_logging_obj=logging_obj, + model="gpt-5.4-mini", + custom_llm_provider="azure_ai", + endpoint="gpt/openai/responses", + ) + info = litellm.get_model_info("azure_ai/gpt-5.4-mini") + + assert response is not None + assert response.usage.output_tokens == 100 + assert logging_obj.call_type == "aresponses" + assert logging_obj._response_cost_calculator(result=response) == pytest.approx( + 1000 * info["input_cost_per_token"] + 100 * info["output_cost_per_token"] + ) From d202885f8b4ce1b0d56b7757ebb0c5849aeb3cb8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:21:45 -0700 Subject: [PATCH 088/411] fix(proxy): run prompt caching counts and custom tokenizer loads off the event loop --- litellm/proxy/proxy_server.py | 4 +- .../prompt_caching_deployment_check.py | 5 +- litellm/utils.py | 10 +-- .../test_custom_tokenizer_bug.py | 10 ++- .../litellm_core_utils/event_loop_lag.py | 1 + tests/test_litellm/proxy/test_proxy_server.py | 37 +++++++++++ .../test_prompt_caching_deployment_check.py | 62 +++++++++++++++++++ 7 files changed, 111 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0915b8dd1b9..77354e9885f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12713,7 +12713,9 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False) CustomHuggingfaceTokenizer | None, model_info.get("custom_tokenizer", None), ) - _tokenizer_used: Final = litellm.utils._select_tokenizer(model=model_to_use, custom_tokenizer=custom_tokenizer) + _tokenizer_used: Final = await asyncify(litellm.utils._select_tokenizer)( + model=model_to_use, custom_tokenizer=custom_tokenizer + ) tokenizer_used: Final = str(_tokenizer_used["type"]) system_message: Final = _system_message(system) diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index 70362e60495..9ea528b9700 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -14,6 +14,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, ) from litellm.integrations.custom_logger import CustomLogger, Span +from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes, StandardLoggingPayload from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt @@ -61,7 +62,7 @@ class PromptCachingDeploymentCheck(CustomLogger): if request_kwargs is not None and request_kwargs.get("_target_order") is not None: return healthy_deployments - if messages is not None and is_prompt_caching_valid_prompt( + if messages is not None and await asyncify(is_prompt_caching_valid_prompt)( messages=messages, model=model, min_token_count=_get_min_token_count_for_deployments(healthy_deployments), @@ -139,7 +140,7 @@ class PromptCachingDeploymentCheck(CustomLogger): return ## PROMPT CACHING - cache model id, if prompt caching valid prompt + provider - if is_prompt_caching_valid_prompt( + if await asyncify(is_prompt_caching_valid_prompt)( model=model, messages=cast(list[AllMessageValues], messages), ): diff --git a/litellm/utils.py b/litellm/utils.py index d0e11bc9551..ab905381b22 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2284,15 +2284,7 @@ def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: st dict: A dictionary with the tokenizer and its type. """ - try: - tokenizer = Tokenizer.from_pretrained( - identifier, - revision=revision, - auth_token=auth_token, - ) - except Exception as e: - verbose_logger.error("Error creating pretrained tokenizer: %s. Defaulting to version without 'auth_token'.", e) - tokenizer = Tokenizer.from_pretrained(identifier, revision=revision) + tokenizer: Final = Tokenizer.from_pretrained(identifier, revision=revision, token=auth_token) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} diff --git a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py index 89899d3e762..c4b1f4f3afd 100644 --- a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py +++ b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py @@ -23,9 +23,9 @@ from litellm.proxy.proxy_server import token_counter def _fake_hf_tokenizer(num_tokens: int) -> MagicMock: encoding = MagicMock() - encoding.ids = list(range(num_tokens)) + encoding.__len__.return_value = num_tokens tokenizer = MagicMock() - tokenizer.encode.return_value = encoding + tokenizer.encode_batch_fast.return_value = [encoding] return tokenizer @@ -68,13 +68,11 @@ async def test_custom_tokenizer_from_model_info_is_used(monkeypatch): ) ) - mock_tokenizer_cls.from_pretrained.assert_called_once_with( - "my-org/custom-tokenizer", revision="v2", auth_token=None - ) + mock_tokenizer_cls.from_pretrained.assert_called_once_with("my-org/custom-tokenizer", revision="v2", token=None) assert response.tokenizer_type == "huggingface_tokenizer" assert response.request_model == "my-embedding-model" assert response.model_used == "self-hosted-embedder" - assert response.total_tokens > 0 + assert response.total_tokens >= 7 @pytest.mark.asyncio diff --git a/tests/test_litellm/litellm_core_utils/event_loop_lag.py b/tests/test_litellm/litellm_core_utils/event_loop_lag.py index c2bbe2204c7..1cac0365547 100644 --- a/tests/test_litellm/litellm_core_utils/event_loop_lag.py +++ b/tests/test_litellm/litellm_core_utils/event_loop_lag.py @@ -25,6 +25,7 @@ async def timed_with_loop_lags(run: Callable[[], Awaitable[T]]) -> tuple[T, floa finished: Final = asyncio.Event() async def timed() -> tuple[T, float]: + await asyncio.sleep(0) started: Final = time.perf_counter() try: return await run(), time.perf_counter() - started diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 7fa3d5dd246..6fa99587dbf 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -12877,3 +12877,40 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun assert response.total_tokens > 0 assert_loop_stayed_free(took, lags) + + +async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeypatch): + from tokenizers import Tokenizer + + from litellm import Router + from tests.test_litellm.litellm_core_utils.event_loop_lag import assert_loop_stayed_free, timed_with_loop_lags + + claude_tokenizer: Final = litellm.utils._select_tokenizer("claude-fable-5")["tokenizer"] + + class SlowHubTokenizer: + @staticmethod + def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> Tokenizer: + time.sleep(0.3) + return claude_tokenizer + + monkeypatch.setattr(litellm.utils, "Tokenizer", SlowHubTokenizer) + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_router", + Router( + model_list=[ + { + "model_name": "self-hosted", + "litellm_params": {"model": "openai/self-hosted-model", "api_base": "http://localhost:8080/v1"}, + "model_info": {"custom_tokenizer": {"identifier": "my-org/tokenizer", "revision": "main", "auth_token": None}}, + } + ] + ), + ) + + response, took, lags = await timed_with_loop_lags( + lambda: proxy_server_module.token_counter(TokenCountRequest(model="self-hosted", prompt="count me off the loop")) + ) + + assert response.tokenizer_type == "huggingface_tokenizer" + assert response.total_tokens > 0 + assert_loop_stayed_free(took, lags) diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 030bdfe03e9..333e7b2ff31 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -477,3 +477,65 @@ async def test_wildcard_route_resolves_underlying_model_minimum(local_model_cost assert deployments[0]["litellm_params"]["model"] == "anthropic/claude-opus-4-6" assert _get_min_token_count_for_deployments(deployments) == 4096 + + +@pytest.mark.asyncio +async def test_async_filter_deployments_counts_the_prompt_off_the_event_loop(): + from tests.large_text import text + from tests.test_litellm.litellm_core_utils.event_loop_lag import ( + assert_loop_stayed_free, + timed_with_loop_lags, + warm_tokenizer, + ) + + warm_tokenizer("anthropic/claude-fable-5") + check = PromptCachingDeploymentCheck(cache=DualCache()) + deployments = _deployments("anthropic/claude-fable-5") + messages = cast(List[AllMessageValues], [{"role": "user", "content": text * 100}]) + + result, took, lags = await timed_with_loop_lags( + lambda: check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, healthy_deployments=deployments, messages=messages + ) + ) + + assert result == deployments + assert_loop_stayed_free(took, lags) + + +@pytest.mark.asyncio +async def test_async_log_success_event_counts_the_prompt_off_the_event_loop(): + from tests.large_text import text + from tests.test_litellm.litellm_core_utils.event_loop_lag import ( + assert_loop_stayed_free, + timed_with_loop_lags, + warm_tokenizer, + ) + + warm_tokenizer("anthropic/claude-fable-5") + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + messages = cast( + List[AllMessageValues], + [{"role": "user", "content": [{"type": "text", "text": text * 100, "cache_control": {"type": "ephemeral"}}]}], + ) + standard_logging_object = { + "call_type": "acompletion", + "model": "anthropic/claude-fable-5", + "messages": messages, + "model_id": "dep-1", + } + + _, took, lags = await timed_with_loop_lags( + lambda: check.async_log_success_event( + kwargs={"standard_logging_object": standard_logging_object}, + response_obj=None, + start_time=None, + end_time=None, + ) + ) + + assert await PromptCachingCache(cache=cache).async_get_model_id(messages=messages, tools=None) == { + "model_id": "dep-1" + } + assert_loop_stayed_free(took, lags) From 5706952588ee2b2445e864ce8a85af3339bb138b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:22:51 -0700 Subject: [PATCH 089/411] fix(azure_ai): drop gpt-chat-latest effort levels, test prices via calculator litellm's azure_ai config rejects reasoning_effort for gpt-chat-latest and Azure documents a fixed reasoning level for it, so the entry no longer advertises reasoning_effort_levels. The catalog metadata tests compare cost_per_token and the whisper transcription cost with the entry the calculator read instead of with list-price literals, the pattern #40195 removed --- ...odel_prices_and_context_window_backup.json | 3 - model_prices_and_context_window.json | 3 - ...azure_ai_foundry_catalog_model_metadata.py | 80 +++++++++---------- 3 files changed, 40 insertions(+), 46 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 674a8b98304..7e7d8a9e930 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3591,9 +3591,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, - "reasoning_effort_levels": [ - "medium" - ], "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 674a8b98304..7e7d8a9e930 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3591,9 +3591,6 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, - "reasoning_effort_levels": [ - "medium" - ], "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 19b082edd8a..84d5cd2a7d4 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -1,11 +1,10 @@ -from dataclasses import dataclass from pathlib import Path from typing import Final import pytest from pydantic import TypeAdapter -from litellm import completion_cost, cost_per_token +from litellm import completion_cost, cost_per_token, get_model_info from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.types.utils import TranscriptionResponse @@ -15,31 +14,39 @@ BACKUP_COST_MAP: Final = REPO_ROOT / "litellm" / "model_prices_and_context_windo COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) AZURE_PRICING_PREFIX: Final = "https://azure.microsoft.com/en-us/pricing/details/" A_MILLION: Final = 1_000_000 +AN_HOUR_IN_SECONDS: Final = 3600 - -@dataclass(frozen=True, slots=True) -class TokenPricedCatalogModel: - catalog_name: str - dollars_per_million_input: float - dollars_per_million_output: float - - -TOKEN_PRICED_MODELS: Final = ( - TokenPricedCatalogModel("gpt-chat-latest", 5.0, 30.0), - TokenPricedCatalogModel("codex-mini", 1.5, 6.0), - TokenPricedCatalogModel("model-router", 0.14, 0.0), - TokenPricedCatalogModel("cohere-command-a", 2.5, 10.0), - TokenPricedCatalogModel("grok-4-20-reasoning", 1.25, 2.5), - TokenPricedCatalogModel("grok-4-20-non-reasoning", 1.25, 2.5), +TOKEN_PRICED_NAMES: Final = ( + "gpt-chat-latest", + "codex-mini", + "model-router", + "cohere-command-a", + "grok-4-20-reasoning", + "grok-4-20-non-reasoning", ) GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning") -CATALOG_NAMES: Final = tuple(spec.catalog_name for spec in TOKEN_PRICED_MODELS) + ("whisper",) +CATALOG_NAMES: Final = TOKEN_PRICED_NAMES + ("whisper",) def _cost_map_entry(path: Path, catalog_name: str) -> dict[str, object]: return COST_MAP_ADAPTER.validate_json(path.read_bytes())[f"azure_ai/{catalog_name}"] +def _whisper_transcription_cost(duration_seconds: int) -> float: + transcription: Final = TranscriptionResponse(text="hello") + transcription._hidden_params = { # pyright: ignore[reportPrivateUsage] # TranscriptionResponse exposes no public hidden-params setter + "custom_llm_provider": "azure_ai", + "model": "azure_ai/whisper", + "audio_transcription_duration": duration_seconds, + } + return completion_cost( + completion_response=transcription, + model="azure_ai/whisper", + custom_llm_provider="azure_ai", + call_type="atranscription", + ) + + @pytest.mark.parametrize("catalog_name", CATALOG_NAMES) def test_azure_ai_catalog_name_routes_to_azure_ai(catalog_name: str) -> None: routed_model, provider, _, _ = get_llm_provider(model=f"azure_ai/{catalog_name}") @@ -47,20 +54,22 @@ def test_azure_ai_catalog_name_routes_to_azure_ai(catalog_name: str) -> None: @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_name_costs_a_million_tokens_at_list_price(spec: TokenPricedCatalogModel) -> None: +@pytest.mark.parametrize("catalog_name", TOKEN_PRICED_NAMES) +def test_azure_ai_catalog_name_charges_its_own_entry_per_token(catalog_name: str) -> None: + entry: Final = get_model_info(f"azure_ai/{catalog_name}") prompt_cost, completion_cost_usd = cost_per_token( - model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=A_MILLION + model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=A_MILLION ) - assert prompt_cost == pytest.approx(spec.dollars_per_million_input) - assert completion_cost_usd == pytest.approx(spec.dollars_per_million_output) + assert prompt_cost > 0 + assert prompt_cost == pytest.approx(A_MILLION * entry["input_cost_per_token"]) + assert completion_cost_usd == pytest.approx(A_MILLION * entry["output_cost_per_token"]) @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize("spec", TOKEN_PRICED_MODELS, ids=lambda spec: spec.catalog_name) -def test_azure_ai_catalog_name_prices_the_same_in_any_casing(spec: TokenPricedCatalogModel) -> None: - lowercase_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) - upper_cost = cost_per_token(model=f"azure_ai/{spec.catalog_name.upper()}", prompt_tokens=A_MILLION, completion_tokens=0) +@pytest.mark.parametrize("catalog_name", TOKEN_PRICED_NAMES) +def test_azure_ai_catalog_name_prices_the_same_in_any_casing(catalog_name: str) -> None: + lowercase_cost = cost_per_token(model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0) + upper_cost = cost_per_token(model=f"azure_ai/{catalog_name.upper()}", prompt_tokens=A_MILLION, completion_tokens=0) assert upper_cost == lowercase_cost @@ -80,19 +89,10 @@ def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalo @pytest.mark.usefixtures("local_model_cost_map") def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None: - transcription: Final = TranscriptionResponse(text="hello") - transcription._hidden_params = { # pyright: ignore[reportPrivateUsage] # TranscriptionResponse exposes no public hidden-params setter - "custom_llm_provider": "azure_ai", - "model": "azure_ai/whisper", - "audio_transcription_duration": 3600, - } - cost = completion_cost( - completion_response=transcription, - model="azure_ai/whisper", - custom_llm_provider="azure_ai", - call_type="atranscription", - ) - assert cost == pytest.approx(0.36) + one_second_cost: Final = _whisper_transcription_cost(1) + one_hour_cost: Final = _whisper_transcription_cost(AN_HOUR_IN_SECONDS) + assert one_second_cost > 0 + assert one_hour_cost == pytest.approx(AN_HOUR_IN_SECONDS * one_second_cost) @pytest.mark.parametrize("catalog_name", CATALOG_NAMES) From 3cadf2f8f7120bf10409a353ef08e4cdc6f78b80 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:35:19 -0700 Subject: [PATCH 090/411] test(azure_ai): charge the router fee over cached prompt tokens too --- .../llms/azure_ai/test_azure_ai_cost_calculator.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 7df14b91741..a43fc3332af 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -138,6 +138,18 @@ class TestAzureModelRouterFlatCost: assert prompt_cost == pytest.approx(1000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) assert completion_cost_usd == 0.0 + def test_unmapped_router_deployment_name_charges_the_fee_over_cached_prompt_tokens_too(self) -> None: + usage = Usage( + prompt_tokens=2000, + completion_tokens=800, + total_tokens=2800, + cache_read_input_tokens=500, + cache_creation_input_tokens=200, + ) + prompt_cost, completion_cost_usd = cost_per_token(model="azure-model-router", usage=usage) + assert prompt_cost == pytest.approx(2000 * ROUTER_FEE_PER_TOKEN, rel=1e-9) + assert completion_cost_usd == 0.0 + def test_router_deployment_name_as_both_names_charges_the_fee_once(self) -> None: usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) prompt_cost, completion_cost_usd = cost_per_token( From 13db79da46f41b9775e781ac0df9c1aa14c29f9a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:38:41 -0700 Subject: [PATCH 091/411] fix(azure_ai): price streamed Responses relays from their terminal event A streamed Responses API relay handed the success handler a bare ResponsesAPIResponse, which the streaming assembly step drops, so the relay never reached the spend callbacks. Hand it the terminal response.completed event instead, which the assembly step already converts, and cover the whole flush path with a regression test that fails on the previous tip. --- .../llms/azure/passthrough/transformation.py | 11 +-- .../base_llm/passthrough/transformation.py | 4 +- .../llms/openai/responses/transformation.py | 9 +- litellm/types/llms/openai.py | 3 + .../test_azure_passthrough_transformation.py | 35 ++++++-- ...est_azure_ai_passthrough_transformation.py | 90 +++++++++++++------ 6 files changed, 107 insertions(+), 45 deletions(-) diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index ab7fe412628..85647ae02c9 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -15,7 +15,7 @@ from litellm.llms.base_llm.passthrough.transformation import ( strip_leading_model_segment, ) from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues, ResponsesAPIResponse +from litellm.types.llms.openai import AllMessageValues, ResponsesAPIResponse, ResponsesTerminalEvent from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse @@ -50,18 +50,19 @@ OPENAI_RELAY_SHAPES: Final = ( ) -def logged_responses_stream(all_chunks: Sequence[str], logging_obj: Logging) -> ResponsesAPIResponse | None: +def logged_responses_stream(all_chunks: Sequence[str], logging_obj: Logging) -> ResponsesTerminalEvent | None: + """A streaming logging object assembles the logged response from the terminal event, not from its body.""" from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig - terminal_response: Final = OpenAIResponsesAPIConfig.parse_terminal_response_from_stream_chunks( + terminal_event: Final = OpenAIResponsesAPIConfig.parse_terminal_event_from_stream_chunks( all_chunks=list(all_chunks) ) - if terminal_response is None: + if terminal_event is None: return None logging_obj.call_type = ( RESPONSES_RELAY_SHAPE.call_type.value ) # rebind-ok: routes cost calculation to the relayed shape's pricing path - return terminal_response + return terminal_event class AzurePassthroughConfig(BasePassthroughConfig): diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index dc857dfc808..20180c5cfa2 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -16,14 +16,14 @@ if TYPE_CHECKING: from httpx import URL, Headers, Response from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesTerminalEvent from litellm.types.rerank import RerankResponse from litellm.types.utils import CostResponseTypes, StandardPassThroughResponseObject from ..chat.transformation import BaseLLMException from ..ocr.transformation import OCRResponse - LoggedRelayResponse: TypeAlias = CostResponseTypes | RerankResponse | ResponsesAPIResponse + LoggedRelayResponse: TypeAlias = CostResponseTypes | RerankResponse | ResponsesAPIResponse | ResponsesTerminalEvent RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 926de3e8854..cde399065fd 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -620,15 +620,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return event_pydantic_model.model_construct(**parsed_chunk) @staticmethod - def parse_terminal_response_from_stream_chunks(all_chunks: list[str]) -> ResponsesAPIResponse | None: + def parse_terminal_event_from_stream_chunks(all_chunks: list[str]) -> ResponsesTerminalEvent | None: for chunk_str in reversed(all_chunks): for event_model in (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent): try: - return event_model.model_validate_json(chunk_str.removeprefix("data: ")).response + return event_model.model_validate_json(chunk_str.removeprefix("data: ")) except ValueError: continue return None + @staticmethod + def parse_terminal_response_from_stream_chunks(all_chunks: list[str]) -> ResponsesAPIResponse | None: + terminal_event: Final = OpenAIResponsesAPIConfig.parse_terminal_event_from_stream_chunks(all_chunks) + return None if terminal_event is None else terminal_event.response + @staticmethod def get_event_model_class(event_type: str) -> type[BaseLiteLLMOpenAIResponseObject]: """ diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index b6da9490e01..b7c4371f32f 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1564,6 +1564,9 @@ class ResponseIncompleteEvent(BaseLiteLLMOpenAIResponseObject): response: ResponsesAPIResponse +ResponsesTerminalEvent: TypeAlias = ResponseCompletedEvent | ResponseIncompleteEvent | ResponseFailedEvent + + class ResponsePartAddedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.RESPONSE_PART_ADDED] item_id: str diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 42dd65ea661..6e85b0cdca6 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -9,7 +9,7 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse from litellm.types.utils import EmbeddingResponse, ModelResponse @@ -103,7 +103,9 @@ def _relay_logging_result(model: str, endpoint: str, body, status_code: int = 20 status_code=status_code, headers={"content-type": "application/json"}, content=json.dumps(body).encode("utf-8"), - request=httpx.Request("POST", f"https://my-resource.openai.azure.com/{endpoint}?api-version=2025-04-01-preview"), + request=httpx.Request( + "POST", f"https://my-resource.openai.azure.com/{endpoint}?api-version=2025-04-01-preview" + ), ) result = AzurePassthroughConfig().logging_non_streaming_response( model=model, @@ -291,7 +293,11 @@ def _azure_responses_stream_chunks(terminal_event: str | None = "response.comple "response.output_text.delta", {"type": "response.output_text.delta", "sequence_number": 1, "item_id": "msg_1", "delta": "hi"}, ), - ] + ([(terminal_event, {"type": terminal_event, "sequence_number": 2, "response": RESPONSES_BODY})] if terminal_event else []) + ] + ( + [(terminal_event, {"type": terminal_event, "sequence_number": 2, "response": RESPONSES_BODY})] + if terminal_event + else [] + ) return [line for name, payload in events for line in (f"event: {name}", _sse_line(payload))] @@ -307,10 +313,10 @@ def test_azure_passthrough_streaming_responses_chunks_are_costed_per_token(): ) info = litellm.get_model_info("azure/gpt-4.1-mini") - assert isinstance(response, ResponsesAPIResponse) - assert response.usage.input_tokens == 1000 + assert isinstance(response, ResponseCompletedEvent) + assert response.response.usage.input_tokens == 1000 assert logging_obj.call_type == "aresponses" - assert logging_obj._response_cost_calculator(result=response) == pytest.approx( + assert logging_obj._response_cost_calculator(result=response.response) == pytest.approx( 1000 * info["input_cost_per_token"] + 100 * info["output_cost_per_token"] ) @@ -373,7 +379,10 @@ def test_azure_passthrough_url_strips_the_leading_router_model_segment(): litellm_params={}, ) - assert str(url) == "https://my-resource.openai.azure.com/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2024-10-21" + assert ( + str(url) + == "https://my-resource.openai.azure.com/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2024-10-21" + ) def test_azure_passthrough_url_rewrites_the_model_group_only_as_a_whole_segment(): @@ -386,7 +395,10 @@ def test_azure_passthrough_url_rewrites_the_model_group_only_as_a_whole_segment( litellm_params={"litellm_metadata": {"model_group": "gpt"}}, ) - assert str(url) == "https://my-resource.openai.azure.com/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2024-10-21" + assert ( + str(url) + == "https://my-resource.openai.azure.com/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2024-10-21" + ) @pytest.mark.parametrize( @@ -394,4 +406,9 @@ def test_azure_passthrough_url_rewrites_the_model_group_only_as_a_whole_segment( [({"stream": True}, True), ({"stream": 1}, True), ({"stream": False}, False), ({}, False)], ) def test_azure_passthrough_is_streaming_request_reads_the_stream_flag(request_data, expected): - assert AzurePassthroughConfig().is_streaming_request(endpoint="openai/deployments/x/chat/completions", request_data=request_data) is expected + assert ( + AzurePassthroughConfig().is_streaming_request( + endpoint="openai/deployments/x/chat/completions", request_data=request_data + ) + is expected + ) diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index 0b23da984e8..064be954518 100644 --- a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -6,6 +6,7 @@ import httpx import pytest import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.passthrough.transformation import AzureAIPassthroughConfig from litellm.llms.base_llm.ocr.transformation import OCRResponse @@ -14,6 +15,36 @@ from litellm.types.utils import EmbeddingResponse, ImageResponse, LlmProviders, from litellm.utils import ProviderConfigManager FOUNDRY_BASE = "https://my-resource.services.ai.azure.com" +RESPONSES_COMPLETED_EVENT = { + "type": "response.completed", + "sequence_number": 2, + "response": { + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.4-mini", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + "usage": {"input_tokens": 1000, "output_tokens": 100, "total_tokens": 1100}, + }, +} + + +class _SpendProbe(CustomLogger): + logged_call_type: str | None = None + logged_cost: float | None = None + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.logged_call_type = kwargs["call_type"] + self.logged_cost = kwargs["response_cost"] @pytest.fixture(autouse=True) @@ -87,7 +118,9 @@ def test_api_base_that_already_ends_in_models_is_cut_back_to_the_foundry_root(): def test_full_url_api_base_that_already_ends_with_the_native_path_is_not_doubled(): - model_router_url = "https://my-resource.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions" + model_router_url = ( + "https://my-resource.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions" + ) url, base = AzureAIPassthroughConfig().get_complete_url( api_base=f"{model_router_url}?api-version=2025-01-01-preview", @@ -267,21 +300,29 @@ def test_non_chat_relay_with_a_non_json_body_logs_the_raw_text(): assert _non_chat_logging_result(b"page one", "text/plain") == {"response": "page one"} -def _relay_logging_obj(model: str, api_base: str) -> Logging: +def _relay_logging_obj( + model: str, + api_base: str, + stream: bool = False, + callbacks: list[CustomLogger] | None = None, + endpoint: str = "", +) -> Logging: logging_obj = Logging( model=model, messages=[], - stream=False, + stream=stream, call_type="allm_passthrough_route", start_time=datetime.now(), litellm_call_id="call-1", function_id="fn-1", + dynamic_async_success_callbacks=callbacks, ) logging_obj.update_environment_variables( model=model, litellm_params={"api_base": api_base, "custom_llm_provider": "azure_ai"}, optional_params={}, custom_llm_provider="azure_ai", + endpoint=endpoint, ) return logging_obj @@ -509,31 +550,10 @@ def test_streaming_chat_completion_chunks_are_costed_like_azure(): def test_streaming_responses_chunks_through_a_router_relay_are_costed_like_azure(): - completed = { - "type": "response.completed", - "sequence_number": 2, - "response": { - "id": "resp_1", - "object": "response", - "created_at": 1, - "status": "completed", - "model": "gpt-5.4-mini", - "output": [ - { - "type": "message", - "id": "msg_1", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "hi", "annotations": []}], - } - ], - "usage": {"input_tokens": 1000, "output_tokens": 100, "total_tokens": 1100}, - }, - } logging_obj = _relay_logging_obj("gpt-5.4-mini", FOUNDRY_BASE) response = AzureAIPassthroughConfig().handle_logging_collected_chunks( - all_chunks=["event: response.completed", "data: " + json.dumps(completed)], + all_chunks=["event: response.completed", "data: " + json.dumps(RESPONSES_COMPLETED_EVENT)], litellm_logging_obj=logging_obj, model="gpt-5.4-mini", custom_llm_provider="azure_ai", @@ -542,8 +562,24 @@ def test_streaming_responses_chunks_through_a_router_relay_are_costed_like_azure info = litellm.get_model_info("azure_ai/gpt-5.4-mini") assert response is not None - assert response.usage.output_tokens == 100 + assert response.response.usage.output_tokens == 100 assert logging_obj.call_type == "aresponses" - assert logging_obj._response_cost_calculator(result=response) == pytest.approx( + assert logging_obj._response_cost_calculator(result=response.response) == pytest.approx( 1000 * info["input_cost_per_token"] + 100 * info["output_cost_per_token"] ) + + +async def test_streaming_responses_relay_flush_reaches_the_success_callbacks_with_a_price(): + probe = _SpendProbe() + logging_obj = _relay_logging_obj( + "gpt-5.4-mini", FOUNDRY_BASE, stream=True, callbacks=[probe], endpoint="gpt/openai/responses" + ) + stream = "event: response.completed\ndata: " + json.dumps(RESPONSES_COMPLETED_EVENT) + "\n\n" + + await logging_obj.async_flush_passthrough_collected_chunks( + raw_bytes=[stream.encode()], provider_config=AzureAIPassthroughConfig() + ) + info = litellm.get_model_info("azure_ai/gpt-5.4-mini") + + assert probe.logged_call_type == "allm_passthrough_route" + assert probe.logged_cost == pytest.approx(1000 * info["input_cost_per_token"] + 100 * info["output_cost_per_token"]) From 78893723f7b65cb73d2c5285cb412b25764a3857 Mon Sep 17 00:00:00 2001 From: Oliver Jensen Date: Tue, 8 Sep 2026 13:05:38 +0200 Subject: [PATCH 092/411] fix(auth): drop env-credential hint from 401 when env login is disabled --- litellm/proxy/auth/login_utils.py | 7 ++++++- tests/test_litellm/proxy/auth/test_login_utils.py | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 6ca7ea5074d..c0a76a4fc20 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -362,8 +362,13 @@ async def authenticate_user( code=401, ) else: + env_credentials_hint: Final = ( + "\nCheck 'UI_USERNAME', 'UI_PASSWORD' in .env file" + if is_env_credential_login_enabled(general_settings) + else "" + ) raise ProxyException( - message="Invalid credentials used to access UI.\nCheck 'UI_USERNAME', 'UI_PASSWORD' in .env file", + message=f"Invalid credentials used to access UI.{env_credentials_hint}", type=ProxyErrorTypes.auth_error, param="invalid_credentials", code=401, diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 72cc6e04a12..e209a491b0a 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -186,6 +186,7 @@ async def test_authenticate_user_invalid_credentials(): assert exc_info.value.type == ProxyErrorTypes.auth_error assert exc_info.value.code == "401" assert "Invalid credentials" in exc_info.value.message + assert "UI_USERNAME" in exc_info.value.message @pytest.mark.asyncio @@ -829,6 +830,8 @@ class TestDisableEnvCredentialLogin: assert exc_info.value.type == ProxyErrorTypes.auth_error assert exc_info.value.code == "401" + assert "UI_USERNAME" not in exc_info.value.message + assert "UI_PASSWORD" not in exc_info.value.message @pytest.mark.asyncio async def test_rejects_master_key_fallback_when_disabled(self): From 992ee37258e7f3c9ed13c374610899388bc8d446 Mon Sep 17 00:00:00 2001 From: Oliver Jensen Date: Tue, 8 Sep 2026 13:21:45 +0200 Subject: [PATCH 093/411] Update litellm/proxy/health_endpoints/_health_endpoints.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/proxy/health_endpoints/_health_endpoints.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index e5e4f89234b..747cfa526d9 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1583,16 +1583,6 @@ async def _show_no_redis_warning() -> bool: def _show_env_credential_login_warning() -> bool: - """ - Whether the UI should warn admins that env-credential login is still enabled. - - UI_USERNAME/UI_PASSWORD (or the master key, when UI_PASSWORD is unset) grant - proxy-admin access with a shared static secret: no per-person identity, no - audit trail, no password policy, and it stays valid until the env var or - master key rotates. That is fine for first-time setup, so it is on by - default, but once real admin accounts exist it should be turned off with - `general_settings.disable_env_credential_login`. - """ from litellm.proxy.auth.login_utils import is_env_credential_login_enabled from litellm.proxy.proxy_server import general_settings From cb8daee55f05e3d4ec728c1b3f15e153176721ea Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 8 Sep 2026 13:23:25 +0000 Subject: [PATCH 094/411] chore(registry): absorb #40185 daybreak-blue and #40159 cloudflare rpm limits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 93 ++++++++++++++++--- model_prices_and_context_window.json | 93 ++++++++++++++++--- tests/test_litellm/test_utils.py | 6 ++ 3 files changed, 164 insertions(+), 28 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5de376218b9..ef8da798922 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13792,7 +13792,8 @@ "max_output_tokens": 3072, "max_tokens": 3072, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/meta/llama-2-7b-chat-int8": { "input_cost_per_token": 1.923e-06, @@ -13801,7 +13802,8 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/mistral/mistral-7b-instruct-v0.1": { "input_cost_per_token": 1.923e-06, @@ -13810,7 +13812,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@hf/thebloke/codellama-7b-instruct-awq": { "input_cost_per_token": 1.923e-06, @@ -13819,7 +13822,8 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/openai/gpt-oss-120b": { "input_cost_per_token": 3.5e-07, @@ -13829,6 +13833,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13839,7 +13844,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.2-3b-instruct": { "input_cost_per_token": 5.09e-08, @@ -13848,7 +13854,8 @@ "max_output_tokens": 80000, "max_tokens": 80000, "mode": "chat", - "output_cost_per_token": 3.35e-07 + "output_cost_per_token": 3.35e-07, + "rpm": 300 }, "cloudflare/@cf/meta/llama-guard-3-8b": { "input_cost_per_token": 4.84e-07, @@ -13857,7 +13864,8 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 3e-08 + "output_cost_per_token": 3e-08, + "rpm": 300 }, "cloudflare/@cf/mistral/mistral-7b-instruct-v0.2-lora": { "input_cost_per_token": 0.0, @@ -13866,7 +13874,8 @@ "max_output_tokens": 15000, "max_tokens": 15000, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/moonshotai/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, @@ -13877,6 +13886,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13888,6 +13898,7 @@ "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 4.881e-06, + "rpm": 300, "supports_reasoning": true }, "cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8": { @@ -13897,7 +13908,8 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 2.87e-07 + "output_cost_per_token": 2.87e-07, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.2-1b-instruct": { "input_cost_per_token": 2.7e-08, @@ -13906,7 +13918,8 @@ "max_output_tokens": 60000, "max_tokens": 60000, "mode": "chat", - "output_cost_per_token": 2.01e-07 + "output_cost_per_token": 2.01e-07, + "rpm": 300 }, "cloudflare/@cf/moonshotai/kimi-k2.6": { "cache_read_input_token_cost": 1.6e-07, @@ -13917,6 +13930,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13928,6 +13942,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13938,7 +13953,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast": { "input_cost_per_token": 2.93e-07, @@ -13948,6 +13964,7 @@ "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 2.253e-06, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/ibm-granite/granite-4.0-h-micro": { @@ -13958,6 +13975,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.12e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/qwen/qwen2.5-coder-32b-instruct": { @@ -13967,7 +13985,8 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1e-06 + "output_cost_per_token": 1e-06, + "rpm": 300 }, "cloudflare/@cf/zai-org/glm-5.2": { "cache_read_input_token_cost": 2.6e-07, @@ -13978,6 +13997,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13989,6 +14009,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-06, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13999,7 +14020,8 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.55e-07 + "output_cost_per_token": 5.55e-07, + "rpm": 300 }, "cloudflare/@cf/qwen/qwen3-30b-a3b-fp8": { "input_cost_per_token": 5.09e-08, @@ -14009,6 +14031,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.35e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14019,7 +14042,8 @@ "max_output_tokens": 3500, "max_tokens": 3500, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/google/gemma-4-26b-a4b-it": { "input_cost_per_token": 1e-07, @@ -14029,6 +14053,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14040,6 +14065,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/meta/llama-3.2-11b-vision-instruct": { @@ -14050,6 +14076,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.76e-07, + "rpm": 300, "supports_vision": true }, "cloudflare/@cf/openai/gpt-oss-20b": { @@ -14060,6 +14087,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14071,6 +14099,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 8.5e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/qwen/qwq-32b": { @@ -14081,6 +14110,7 @@ "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 1e-06, + "rpm": 300, "supports_reasoning": true }, "codestral/codestral-2405": { @@ -54901,6 +54931,39 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-daybreak-blue-5.6-sol": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-daybreak-blue-56-sol.html" + }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, @@ -60822,6 +60885,7 @@ "litellm_provider": "cloudflare", "mode": "audio_transcription", "output_cost_per_second": 0.0, + "rpm": 720, "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", "supported_endpoints": [ "/v1/audio/transcriptions" @@ -60832,6 +60896,7 @@ "litellm_provider": "cloudflare", "mode": "audio_transcription", "output_cost_per_second": 0.0, + "rpm": 720, "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", "supported_endpoints": [ "/v1/audio/transcriptions" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5de376218b9..ef8da798922 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13792,7 +13792,8 @@ "max_output_tokens": 3072, "max_tokens": 3072, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/meta/llama-2-7b-chat-int8": { "input_cost_per_token": 1.923e-06, @@ -13801,7 +13802,8 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/mistral/mistral-7b-instruct-v0.1": { "input_cost_per_token": 1.923e-06, @@ -13810,7 +13812,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@hf/thebloke/codellama-7b-instruct-awq": { "input_cost_per_token": 1.923e-06, @@ -13819,7 +13822,8 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.923e-06 + "output_cost_per_token": 1.923e-06, + "rpm": 300 }, "cloudflare/@cf/openai/gpt-oss-120b": { "input_cost_per_token": 3.5e-07, @@ -13829,6 +13833,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13839,7 +13844,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.2-3b-instruct": { "input_cost_per_token": 5.09e-08, @@ -13848,7 +13854,8 @@ "max_output_tokens": 80000, "max_tokens": 80000, "mode": "chat", - "output_cost_per_token": 3.35e-07 + "output_cost_per_token": 3.35e-07, + "rpm": 300 }, "cloudflare/@cf/meta/llama-guard-3-8b": { "input_cost_per_token": 4.84e-07, @@ -13857,7 +13864,8 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 3e-08 + "output_cost_per_token": 3e-08, + "rpm": 300 }, "cloudflare/@cf/mistral/mistral-7b-instruct-v0.2-lora": { "input_cost_per_token": 0.0, @@ -13866,7 +13874,8 @@ "max_output_tokens": 15000, "max_tokens": 15000, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/moonshotai/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, @@ -13877,6 +13886,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13888,6 +13898,7 @@ "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 4.881e-06, + "rpm": 300, "supports_reasoning": true }, "cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8": { @@ -13897,7 +13908,8 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 2.87e-07 + "output_cost_per_token": 2.87e-07, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.2-1b-instruct": { "input_cost_per_token": 2.7e-08, @@ -13906,7 +13918,8 @@ "max_output_tokens": 60000, "max_tokens": 60000, "mode": "chat", - "output_cost_per_token": 2.01e-07 + "output_cost_per_token": 2.01e-07, + "rpm": 300 }, "cloudflare/@cf/moonshotai/kimi-k2.6": { "cache_read_input_token_cost": 1.6e-07, @@ -13917,6 +13930,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13928,6 +13942,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13938,7 +13953,8 @@ "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast": { "input_cost_per_token": 2.93e-07, @@ -13948,6 +13964,7 @@ "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 2.253e-06, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/ibm-granite/granite-4.0-h-micro": { @@ -13958,6 +13975,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.12e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/qwen/qwen2.5-coder-32b-instruct": { @@ -13967,7 +13985,8 @@ "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1e-06 + "output_cost_per_token": 1e-06, + "rpm": 300 }, "cloudflare/@cf/zai-org/glm-5.2": { "cache_read_input_token_cost": 2.6e-07, @@ -13978,6 +13997,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, + "rpm": 20, "supports_function_calling": true, "supports_reasoning": true }, @@ -13989,6 +14009,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-06, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -13999,7 +14020,8 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.55e-07 + "output_cost_per_token": 5.55e-07, + "rpm": 300 }, "cloudflare/@cf/qwen/qwen3-30b-a3b-fp8": { "input_cost_per_token": 5.09e-08, @@ -14009,6 +14031,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.35e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14019,7 +14042,8 @@ "max_output_tokens": 3500, "max_tokens": 3500, "mode": "chat", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "rpm": 300 }, "cloudflare/@cf/google/gemma-4-26b-a4b-it": { "input_cost_per_token": 1e-07, @@ -14029,6 +14053,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14040,6 +14065,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/meta/llama-3.2-11b-vision-instruct": { @@ -14050,6 +14076,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.76e-07, + "rpm": 300, "supports_vision": true }, "cloudflare/@cf/openai/gpt-oss-20b": { @@ -14060,6 +14087,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, + "rpm": 300, "supports_function_calling": true, "supports_reasoning": true }, @@ -14071,6 +14099,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 8.5e-07, + "rpm": 300, "supports_function_calling": true }, "cloudflare/@cf/qwen/qwq-32b": { @@ -14081,6 +14110,7 @@ "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 1e-06, + "rpm": 300, "supports_reasoning": true }, "codestral/codestral-2405": { @@ -54901,6 +54931,39 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-daybreak-blue-5.6-sol": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-daybreak-blue-56-sol.html" + }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, @@ -60822,6 +60885,7 @@ "litellm_provider": "cloudflare", "mode": "audio_transcription", "output_cost_per_second": 0.0, + "rpm": 720, "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", "supported_endpoints": [ "/v1/audio/transcriptions" @@ -60832,6 +60896,7 @@ "litellm_provider": "cloudflare", "mode": "audio_transcription", "output_cost_per_second": 0.0, + "rpm": 720, "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", "supported_endpoints": [ "/v1/audio/transcriptions" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 8a56a84ade7..cfdbbcb1132 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -54,6 +54,12 @@ from litellm.utils import ( # Adds the parent directory to the system path +def test_cloudflare_model_info_includes_rpm(local_model_cost_map): + assert litellm.get_model_info("cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8")["rpm"] == 300 + assert litellm.get_model_info("cloudflare/@cf/moonshotai/kimi-k2.6")["rpm"] == 20 + assert litellm.get_model_info("cloudflare/@cf/openai/whisper-large-v3-turbo")["rpm"] == 720 + + def test_get_utc_datetime_returns_current_aware_utc_time() -> None: before: Final = datetime.now(timezone.utc) result: Final = litellm.utils.get_utc_datetime() From fd1fad5e0559db253842fdb3972a94081c58734d Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 8 Sep 2026 13:37:56 +0000 Subject: [PATCH 095/411] test: annotate registry metadata test parameters Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../responses/test_chatgpt_responses_transformation.py | 6 ++++-- tests/test_litellm/test_utils.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 8a0027fdb53..a7520bd5955 100644 --- a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -54,7 +54,7 @@ class TestChatGPTResponsesAPITransformation: "chatgpt/gpt-5.6-terra", ], ) - def test_chatgpt_responses_model_metadata(self, model_name, local_model_cost_map): + def test_chatgpt_responses_model_metadata(self, model_name: str, local_model_cost_map: None) -> None: model_info = litellm.get_model_info(model_name) assert model_info["litellm_provider"] == "chatgpt" @@ -75,7 +75,9 @@ class TestChatGPTResponsesAPITransformation: "gpt-5.6-terra", ], ) - def test_chatgpt_models_bridge_chat_completions_to_responses(self, model_name, local_model_cost_map): + def test_chatgpt_models_bridge_chat_completions_to_responses( + self, model_name: str, local_model_cost_map: None + ) -> None: """A chat completions request for these models must take the Responses bridge. `gpt-5.6-*` also exists as an openai chat model, so an unregistered diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index cfdbbcb1132..45532f91f16 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -54,7 +54,7 @@ from litellm.utils import ( # Adds the parent directory to the system path -def test_cloudflare_model_info_includes_rpm(local_model_cost_map): +def test_cloudflare_model_info_includes_rpm(local_model_cost_map: None) -> None: assert litellm.get_model_info("cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8")["rpm"] == 300 assert litellm.get_model_info("cloudflare/@cf/moonshotai/kimi-k2.6")["rpm"] == 20 assert litellm.get_model_info("cloudflare/@cf/openai/whisper-large-v3-turbo")["rpm"] == 720 From 17dd6afbaafed87e6e7380278ca13a543046160a Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 8 Sep 2026 13:51:58 +0000 Subject: [PATCH 096/411] fix: keep reasoning_effort for mode: responses bridge deployments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/main.py | 12 +++++++++++- tests/test_litellm/test_main.py | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 75b7f7f10a5..062ddb9a293 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5398,6 +5398,16 @@ def completion( if dynamic_api_key is not None: api_key = dynamic_api_key # check if user passed in any of the OpenAI optional params + allowed_openai_params: Final[list[str] | None] = cast( + list[str] | None, + ( + [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] + if responses_api_model_info.get("mode") == "responses" + and not skip_responses_api_bridge + and "reasoning_effort" not in (kwargs.get("allowed_openai_params") or []) + else kwargs.get("allowed_openai_params") + ), + ) optional_param_args: Final = { "functions": functions, "function_call": function_call, @@ -5442,7 +5452,7 @@ def completion( "service_tier": service_tier, "store": store, "prompt_cache_key": prompt_cache_key, - "allowed_openai_params": kwargs.get("allowed_openai_params"), + "allowed_openai_params": allowed_openai_params, "base_model": base_model, } optional_params = get_optional_params(**optional_param_args, **non_default_params) diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 038df3656fe..9ce2bb8400d 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -1367,6 +1367,40 @@ def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( } +@pytest.mark.parametrize("reasoning_effort", ["high", {"effort": "high"}]) +@patch("litellm.completion_extras.responses_api_bridge.completion") +def test_responses_bridge_preserves_reasoning_effort_with_drop_params( + mock_responses_completion, reasoning_effort, restore_model_registry +): + mock_responses_completion.return_value = MagicMock() + model = "perplexity/test-responses-bridge" + litellm.register_model( + { + model: { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_reasoning": True, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + + with patch.object(litellm, "supports_reasoning", return_value=False): + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=reasoning_effort, + drop_params=True, + api_key="fake-key", + api_base="https://api.perplexity.ai", + ) + + optional_params = mock_responses_completion.call_args.kwargs["optional_params"] + assert optional_params["reasoning_effort"] == reasoning_effort + + @pytest.mark.parametrize( "model, model_info, expected_model_param, expected_base_model_param", [ From 5520c3bb1a768263bbc4458b14da7d87e3696a41 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 8 Sep 2026 13:53:02 +0000 Subject: [PATCH 097/411] refactor: simplify allowed_openai_params bridge override Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/main.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 062ddb9a293..01fe6a7c674 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5398,15 +5398,13 @@ def completion( if dynamic_api_key is not None: api_key = dynamic_api_key # check if user passed in any of the OpenAI optional params - allowed_openai_params: Final[list[str] | None] = cast( - list[str] | None, - ( - [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] - if responses_api_model_info.get("mode") == "responses" - and not skip_responses_api_bridge - and "reasoning_effort" not in (kwargs.get("allowed_openai_params") or []) - else kwargs.get("allowed_openai_params") - ), + bridges_to_responses_api: Final = ( + responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge + ) + allowed_openai_params: Final[list[str] | None] = ( + [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] + if bridges_to_responses_api + else kwargs.get("allowed_openai_params") ) optional_param_args: Final = { "functions": functions, From 6f3b3c7957a67d45542538bcda6dc689752b7316 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 8 Sep 2026 14:27:12 +0000 Subject: [PATCH 098/411] test: cover responses bridge reasoning at HTTP boundary Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_main.py | 70 +++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 9ce2bb8400d..8a9d5e07258 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -1368,18 +1368,57 @@ def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( @pytest.mark.parametrize("reasoning_effort", ["high", {"effort": "high"}]) -@patch("litellm.completion_extras.responses_api_bridge.completion") def test_responses_bridge_preserves_reasoning_effort_with_drop_params( - mock_responses_completion, reasoning_effort, restore_model_registry + reasoning_effort, + restore_model_registry, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, ): - mock_responses_completion.return_value = MagicMock() - model = "perplexity/test-responses-bridge" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + response_body: Final = { + "id": "resp_test", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "test-responses-bridge", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "usage": { + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "truncation": None, + "user": None, + } + response_route: Final = respx_mock.post("https://api.perplexity.ai/v1/responses").respond(json=response_body) + model: Final = "perplexity/test-responses-bridge" litellm.register_model( { model: { "litellm_provider": "perplexity", "mode": "responses", - "supports_reasoning": True, + "supports_reasoning": False, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, } @@ -1387,18 +1426,17 @@ def test_responses_bridge_preserves_reasoning_effort_with_drop_params( persist_across_reloads=False, ) - with patch.object(litellm, "supports_reasoning", return_value=False): - litellm.completion( - model=model, - messages=[{"role": "user", "content": "hello"}], - reasoning_effort=reasoning_effort, - drop_params=True, - api_key="fake-key", - api_base="https://api.perplexity.ai", - ) + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=reasoning_effort, + drop_params=True, + api_key="fake-key", + api_base="https://api.perplexity.ai", + ) - optional_params = mock_responses_completion.call_args.kwargs["optional_params"] - assert optional_params["reasoning_effort"] == reasoning_effort + request_body: Final = json.loads(response_route.calls[0].request.content) + assert request_body["reasoning"] == {"effort": "high"} @pytest.mark.parametrize( From 06497ab42a1ad055e8cbb7695d778cd7a4158ea5 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 8 Sep 2026 15:13:53 +0000 Subject: [PATCH 099/411] fix(ui): guard playground cost metric against null and NaN Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat_ui/ResponseMetrics.test.tsx | 19 ++++++++++++ .../components/chat_ui/ResponseMetrics.tsx | 2 +- .../llm_calls/chat_completion.test.tsx | 6 ++++ .../components/llm_calls/chat_completion.tsx | 5 +++- .../llm_calls/responses_api.test.tsx | 29 +++++++++++++++++++ .../components/llm_calls/responses_api.tsx | 5 +++- 6 files changed, 63 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx index f31e1839739..4ba6d8d50b4 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx @@ -51,4 +51,23 @@ describe("ResponseMetrics prompt cache chips", () => { expect(screen.queryByText(/Response Cache/)).not.toBeInTheDocument(); }); + + it("does not render the Cost chip when a persisted cost is null", () => { + render(); + + expect(screen.queryByText(/Cost:/)).not.toBeInTheDocument(); + expect(screen.getByText("In: 1")).toBeInTheDocument(); + }); + + it("does not render the Cost chip for NaN", () => { + render(); + + expect(screen.queryByText(/Cost:/)).not.toBeInTheDocument(); + }); + + it("renders the Cost chip for a finite cost", () => { + render(); + + expect(screen.getByText("Cost: $0.000063")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx index ec62d0618d7..bb7debc7aee 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx @@ -159,7 +159,7 @@ const ResponseMetrics: React.FC = ({ timeToFirstToken, tot /> )} - {usage?.cost !== undefined && ( + {typeof usage?.cost === "number" && Number.isFinite(usage.cost) && ( { expect(usageData).not.toHaveProperty("cacheReadTokens"); expect(usageData).not.toHaveProperty("cacheCreationTokens"); }); + + it("omits cost when the provider reports a non-numeric value", async () => { + const usageData = await captureUsage({ cost: "not-a-number" }); + + expect(usageData).toEqual(expect.not.objectContaining({ cost: expect.anything() })); + }); }); describe("chat_completion response cache", () => { diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx index cd0852bc06e..b813a35f687 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx @@ -245,7 +245,10 @@ export async function makeOpenAIChatCompletionRequest( // Extract cost from usage object if available if (chunkWithUsage.usage.cost !== undefined && chunkWithUsage.usage.cost !== null) { - usageData.cost = parseFloat(chunkWithUsage.usage.cost); + const parsedCost = parseFloat(chunkWithUsage.usage.cost); + if (Number.isFinite(parsedCost)) { + usageData.cost = parsedCost; + } } onUsageData(usageData); diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 0b94c093acd..7b13f0ac2e3 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -231,6 +231,35 @@ describe("responses_api", () => { expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ cost: expect.anything() }), ""); }); + it("should omit cost when the proxy reports a non-numeric cost", async () => { + async function* streamWithNonNumericCost() { + yield { + type: "response.completed", + response: { + id: "resp_non_numeric_cost", + usage: { output_tokens: 12, input_tokens: 12, total_tokens: 24, cost: "not-a-number" }, + }, + }; + } + mockResponsesCreate.mockResolvedValueOnce(streamWithNonNumericCost()); + + const onUsageData = vi.fn(); + + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, + undefined, + undefined, + undefined, + onUsageData, + ); + + expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ cost: expect.anything() }), ""); + }); + it("should replay MCP output items as events for a non-streaming response", async () => { mockResponsesCreate.mockReturnValueOnce( nonStreamingResponse({ diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index 7ab76488504..63085a81b94 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -312,7 +312,10 @@ export async function makeOpenAIResponsesRequest( } if (usage.cost !== undefined && usage.cost !== null) { - usageData.cost = Number(usage.cost); + const parsedCost = Number(usage.cost); + if (Number.isFinite(parsedCost)) { + usageData.cost = parsedCost; + } } onUsageData(usageData, mcpToolUsed); From cbe340a31ca81c40be088520ebbaaaca644977ff Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:38:18 -0700 Subject: [PATCH 100/411] feat(guardrails): deliver tool-call rewrites into buffered streams A post_call pipeline guardrail that rewrites a streamed tool call (its arguments or its name) now has that rewrite written back across the buffered chunks on chat, Responses, and Messages streams, so the client receives the rewritten tool call instead of the original. The chat handler rewrites the first fragment of each tool-call index and blanks the rest, the Responses handler syncs the function_call output items and their argument events, and the Messages handler rewrites the tool_use content_block_start and input_json_delta events in both dict and SSE-bytes chunks. The delivers_ended_stream_text_rewrites flag becomes delivers_ended_stream_rewrites, since the write-back now covers both text and tool calls, and the executor only discards a tool-call rewrite on translations without write-back or on a shape the translation refuses. --- .../chat/guardrail_translation/handler.py | 176 +++++++++++++++--- .../guardrail_translation/base_translation.py | 17 +- .../chat/guardrail_translation/handler.py | 96 +++++++++- .../guardrail_translation/handler.py | 111 ++++++++++- .../proxy/policy_engine/pipeline_executor.py | 22 +-- litellm/proxy/utils.py | 10 +- .../test_anthropic_guardrail_handler.py | 66 +++++++ .../test_openai_guardrail_handler.py | 73 ++++++++ ...test_openai_responses_guardrail_handler.py | 88 +++++++++ .../policy_engine/test_pipeline_executor.py | 22 ++- .../proxy_logging/test_guardrail_pipeline.py | 9 +- 11 files changed, 618 insertions(+), 72 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index e486be12fe2..2049866b444 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -13,10 +13,11 @@ Pattern Overview: """ import json -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass from itertools import chain, repeat +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable from typing_extensions import ReadOnly, TypedDict, assert_never @@ -41,6 +42,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( merge_guardrailed_scoped_messages, merge_returned_tools_into_request_tools, scoped_structured_message_indices, + stream_item_field, stream_item_fingerprint, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( @@ -153,6 +155,28 @@ class ExtractedInput: EMPTY_EXTRACTED_INPUT: Final = ExtractedInput(scanned=(), images=()) +@dataclass(frozen=True, slots=True) +class _ToolCallShape: + name: str | None + arguments: str + + +_SSEEventRewriter = Callable[[Mapping[str, object]], Mapping[str, object] | None] + + +def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...]: + """The guardrail-visible shape of each tool call, whether the guardrail handed + back the ``ChatCompletionMessageToolCall`` objects it was given or plain dicts.""" + functions: Final = tuple(stream_item_field(tool_call, "function") for tool_call in tool_calls) + return tuple( + _ToolCallShape( + name=name if isinstance(name := stream_item_field(function, "name"), str) else None, + arguments=arguments if isinstance(arguments := stream_item_field(function, "arguments"), str) else "", + ) + for function in functions + ) + + class _AnthropicSSEDelta(TypedDict, total=False): type: ReadOnly[str] text: ReadOnly[str] @@ -170,7 +194,7 @@ class AnthropicMessagesHandler(BaseTranslation): them through guardrail rewrites; downstream provider handling is out of scope. """ - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True def __init__(self): super().__init__() @@ -1050,6 +1074,7 @@ class AnthropicMessagesHandler(BaseTranslation): first_choice.message.tool_calls, ) string_so_far = first_choice.message.content + pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ()) guardrail_inputs: Final = GenericGuardrailAPIInputs() if string_so_far: guardrail_inputs["texts"] = [string_so_far] @@ -1084,6 +1109,19 @@ class AnthropicMessagesHandler(BaseTranslation): and guardrailed_texts[0] != string_so_far ): self._write_ended_stream_text_rewrite(responses_so_far, guardrailed_texts[0]) + if deliver_ended_stream_rewrites: + returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls") + self._write_ended_stream_tool_call_rewrites( + responses_so_far, + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + post_guardrail_tool_calls=_tool_call_shapes( + returned_tool_calls + if isinstance(returned_tool_calls, list) + and len(returned_tool_calls) == len(pre_guardrail_tool_calls) + else tool_calls_list or () + ), + guardrail_name=guardrail_to_apply.guardrail_name or "unknown", + ) else: verbose_proxy_logger.debug("Skipping output guardrail - model response has no choices") return responses_so_far @@ -1212,38 +1250,120 @@ class AnthropicMessagesHandler(BaseTranslation): """Deliver an ended-stream guardrail text rewrite by rewriting the buffered chunks in place: the first ``text_delta`` carries the full rewritten text and every later one is blanked, leaving the surrounding - message and content-block framing untouched. Handles both chunk formats - this stream carries (parsed event dicts and raw SSE bytes).""" + message and content-block framing untouched.""" replacements: Final = chain((rewritten_text,), repeat("")) - for idx, item in enumerate(responses_so_far): - if isinstance(item, dict): - delta = item.get("delta") - if item.get("type") == "content_block_delta" and isinstance(delta, dict): - if delta.get("type") == "text_delta": - delta["text"] = next(replacements) - elif isinstance(item, (bytes, bytearray)): - responses_so_far[idx] = ( # rebind-ok: delivers the rewrite into the caller's buffer - AnthropicMessagesHandler._rewrite_sse_text_deltas(bytes(item), replacements) - ) + + def rewrite_text_delta(event: Mapping[str, object]) -> Mapping[str, object] | None: + delta: Final = event.get("delta") + if event.get("type") != "content_block_delta" or not isinstance(delta, Mapping): + return None + if delta.get("type") != "text_delta": + return None + return {**event, "delta": {**delta, "text": next(replacements)}} + + AnthropicMessagesHandler._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta) + + @classmethod + def _write_ended_stream_tool_call_rewrites( + cls, + responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + *, + pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], + post_guardrail_tool_calls: tuple[_ToolCallShape, ...], + guardrail_name: str, + ) -> None: + """Deliver ended-stream guardrail tool-call rewrites by rewriting the + buffered chunks in place: the rebuilt response lists tool calls in the + order of the stream's ``tool_use`` blocks, so the nth rewritten call lands + on the nth block, its first ``input_json_delta`` carrying the full rewritten + arguments, every later one blanked, and ``content_block_start`` carrying the + rewritten name. Blocks that do not line up with the rebuilt tool calls make + the rewrite undeliverable, so the pipeline executor discards it and releases + the original chunks.""" + if post_guardrail_tool_calls == pre_guardrail_tool_calls: + return + block_indices: Final = tuple( + index + for item in responses_so_far + for event in cls._iter_sse_events(item) + if event.get("type") == "content_block_start" + and isinstance(block := event.get("content_block"), Mapping) + and block.get("type") == "tool_use" + and isinstance(index := event.get("index"), int) + ) + if len(block_indices) != len(post_guardrail_tool_calls): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + raise UndeliverableStreamRewrite(guardrail_name) + rewrites_by_block: Final = MappingProxyType( + { + index: after + for index, before, after in zip(block_indices, pre_guardrail_tool_calls, post_guardrail_tool_calls) + if after != before + } + ) + argument_replacements: Final = MappingProxyType( + {index: chain((rewrite.arguments,), repeat("")) for index, rewrite in rewrites_by_block.items()} + ) + + def rewrite_tool_use(event: Mapping[str, object]) -> Mapping[str, object] | None: + index: Final = event.get("index") + if not isinstance(index, int) or index not in rewrites_by_block: + return None + match event.get("type"): + case "content_block_start": + block: Final = event.get("content_block") + name: Final = rewrites_by_block[index].name + if not isinstance(block, Mapping) or name is None: + return None + return {**event, "content_block": {**block, "name": name}} + case "content_block_delta": + delta: Final = event.get("delta") + if not isinstance(delta, Mapping) or delta.get("type") != "input_json_delta": + return None + return {**event, "delta": {**delta, "partial_json": next(argument_replacements[index])}} + case _: + return None + + cls._rewrite_ended_stream_events(responses_so_far, rewrite_tool_use) @staticmethod - def _rewrite_sse_text_deltas(sse_bytes: bytes, replacements: "Iterator[str]") -> bytes: - """Rewrite every ``text_delta`` data line in one SSE chunk with the next - replacement text, leaving all other events and framing byte-identical.""" + def _rewrite_ended_stream_events( + responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + rewrite_event: _SSEEventRewriter, + ) -> None: + """Replace every buffered event ``rewrite_event`` returns a rewrite for, in + both chunk formats this stream carries (parsed event dicts and raw SSE + bytes), leaving every other event and the framing untouched.""" + rewritten_items: Final = tuple( + AnthropicMessagesHandler._rewrite_buffered_item(item, rewrite_event) for item in responses_so_far + ) + responses_so_far[:] = rewritten_items # rebind-ok: delivers the rewrites into the caller's buffer + + @staticmethod + def _rewrite_buffered_item(item: object, rewrite_event: _SSEEventRewriter) -> object: + if isinstance(item, dict): + rewritten: Final = rewrite_event(_as_str_mapping(item)) + return item if rewritten is None else dict(rewritten) + if isinstance(item, (bytes, bytearray)): + return AnthropicMessagesHandler._rewrite_sse_events(bytes(item), rewrite_event) + return item + + @staticmethod + def _rewrite_sse_events(sse_bytes: bytes, rewrite_event: _SSEEventRewriter) -> bytes: + """Rewrite the data lines of one SSE chunk that ``rewrite_event`` rewrites, + leaving all other events and framing byte-identical.""" try: decoded: Final = sse_bytes.decode("utf-8") except UnicodeDecodeError: return sse_bytes return "\n\n".join( - AnthropicMessagesHandler._rewrite_sse_block(block, replacements) for block in decoded.split("\n\n") + "\n".join(AnthropicMessagesHandler._rewrite_sse_line(line, rewrite_event) for line in block.split("\n")) + for block in decoded.split("\n\n") ).encode("utf-8") @staticmethod - def _rewrite_sse_block(block: str, replacements: "Iterator[str]") -> str: - return "\n".join(AnthropicMessagesHandler._rewrite_sse_line(line, replacements) for line in block.split("\n")) - - @staticmethod - def _rewrite_sse_line(line: str, replacements: "Iterator[str]") -> str: + def _rewrite_sse_line(line: str, rewrite_event: _SSEEventRewriter) -> str: if not line.startswith("data:"): return line try: @@ -1252,14 +1372,10 @@ class AnthropicMessagesHandler(BaseTranslation): ) except json.JSONDecodeError: return line - if not isinstance(data, dict) or data.get("type") != "content_block_delta": + if not isinstance(data, dict): return line - delta: Final = data.get("delta") - if not isinstance(delta, dict) or delta.get("type") != "text_delta": - return line - return "data: " + json.dumps( - {**data, "delta": {**delta, "text": next(replacements)}} # mutable-ok: json.dumps needs plain dicts - ) + rewritten: Final = rewrite_event(_as_str_mapping(data)) + return line if rewritten is None else "data: " + json.dumps(rewritten) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: stream_ended: Final = self._check_streaming_has_ended(responses_so_far) diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index afd8e0f67f7..6d1a9ab1c3e 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -52,13 +52,14 @@ class StreamingScanKey: class BaseTranslation(ABC): - delivers_ended_stream_text_rewrites: ClassVar[bool] = False + delivers_ended_stream_rewrites: ClassVar[bool] = False """Whether ``process_output_streaming_response`` accepts ``deliver_ended_stream_rewrites=True`` and, on an ended (fully buffered) - stream, writes guardrail text rewrites back across ``responses_so_far`` so - a buffered pipeline can release rewritten chunks. Tool-call rewrites, and - text rewrites on every other translation, are undeliverable: the pipeline - executor discards them and releases the original chunks.""" + stream, writes guardrail text and tool-call rewrites back across + ``responses_so_far`` so a buffered pipeline can release rewritten chunks, + raising ``UndeliverableStreamRewrite`` for a shape it cannot place. Rewrites + on every other translation are undeliverable: the pipeline executor + discards them and releases the original chunks.""" @staticmethod def transform_user_api_key_dict_to_metadata( @@ -175,9 +176,9 @@ class BaseTranslation(ABC): transformations (see ``StreamTransformSink``); base handlers ignore it. ``deliver_ended_stream_rewrites`` is passed True only when the caller holds the whole buffered stream and the subclass declares - ``delivers_ended_stream_text_rewrites``: the handler then writes - guardrail text rewrites back across ``responses_so_far`` instead of - discarding them. + ``delivers_ended_stream_rewrites``: the handler then writes + guardrail text and tool-call rewrites back across ``responses_so_far`` + instead of discarding them. """ return responses_so_far diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 80292aef2cf..42c95ac3316 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -49,6 +49,8 @@ from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import coerce_stream_holdback_value, ) from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + ChatCompletionMessageToolCall, Choices, GenericGuardrailAPIInputs, ModelResponse, @@ -78,7 +80,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): Methods can be overridden to customize behavior for different message formats. """ - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: """ @@ -610,13 +612,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): deliver_ended_stream_rewrites: bool, ) -> None: """Ended-stream path: rebuild the full response, run the non-streaming - output guardrail against it, and (when opted in) write any text rewrite - back across the buffered chunks.""" + output guardrail against it, and (when opted in) write any text or + tool-call rewrite back across the buffered chunks.""" model_response: Final = cast( ModelResponse, stream_chunk_builder(chunks=responses_so_far, logging_obj=litellm_logging_obj), ) pre_guardrail_texts: Final = self._string_choice_contents(model_response) + pre_guardrail_tool_calls: Final = self._function_tool_call_shapes(model_response) await self.process_output_response( response=model_response, guardrail_to_apply=guardrail_to_apply, @@ -624,13 +627,21 @@ class OpenAIChatCompletionsHandler(BaseTranslation): user_api_key_dict=user_api_key_dict, request_data=request_data, ) - if deliver_ended_stream_rewrites: - await self._write_ended_stream_text_rewrites( - responses_so_far=responses_so_far, - guardrailed_response=model_response, - pre_guardrail_texts=pre_guardrail_texts, - guardrail_name=guardrail_to_apply.guardrail_name or "unknown", - ) + if not deliver_ended_stream_rewrites: + return + guardrail_name: Final = guardrail_to_apply.guardrail_name or "unknown" + await self._write_ended_stream_text_rewrites( + responses_so_far=responses_so_far, + guardrailed_response=model_response, + pre_guardrail_texts=pre_guardrail_texts, + guardrail_name=guardrail_name, + ) + self._write_ended_stream_tool_call_rewrites( + responses_so_far=responses_so_far, + guardrailed_response=model_response, + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + guardrail_name=guardrail_name, + ) def build_stream_error_items( self, @@ -1043,6 +1054,71 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=[(target_choice_index, None) for _ in changed], # mutable-ok: callee takes lists ) + @staticmethod + def _function_tool_call_shapes(response: "ModelResponse") -> tuple[tuple[str | None, str], ...]: + return tuple( + (tool_call.function.name, tool_call.function.arguments) + for choice in response.choices + for tool_call in choice.message.tool_calls or () + if isinstance(tool_call, ChatCompletionMessageToolCall) + ) + + @staticmethod + def _function_tool_call_fragments( + responses_so_far: Sequence["ModelResponseStream"], + ) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]: + """Group the stream's function tool-call fragments by their tool-call index, in + the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping + only the indices the builder keeps (an id and a name somewhere in the stream).""" + fragments: Final = tuple( + tool_call + for response in responses_so_far + for choice in response.choices + for tool_call in choice.delta.tool_calls or () + if isinstance(tool_call, ChatCompletionDeltaToolCall) + ) + identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id) + named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name) + return tuple( + tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named) + ) + + def _write_ended_stream_tool_call_rewrites( + self, + responses_so_far: list["ModelResponseStream"], # mutable-ok: rewrites the caller's buffered chunks in place + guardrailed_response: "ModelResponse", + pre_guardrail_tool_calls: tuple[tuple[str | None, str], ...], + guardrail_name: str, + ) -> None: + """Write ended-stream guardrail tool-call rewrites back across the buffered + chunks: the rewritten name and full arguments land in the tool call's first + fragment and the arguments of its later fragments are blanked, mirroring the + text write-back. A rewrite on a stream carrying more than one distinct choice + index, or whose fragments do not line up with the rebuilt tool calls, is + reported as undeliverable, so the pipeline executor discards it and releases + the original chunks.""" + post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response) + if post_guardrail_tool_calls == pre_guardrail_tool_calls: + return + stream_choice_indices: Final = frozenset( + choice.index for response in responses_so_far for choice in response.choices + ) + fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far) + if len(stream_choice_indices) != 1 or len(fragments_by_tool_call) != len(post_guardrail_tool_calls): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + raise UndeliverableStreamRewrite(guardrail_name) + for before, (name, arguments), fragments in zip( + pre_guardrail_tool_calls, post_guardrail_tool_calls, fragments_by_tool_call + ): + if (name, arguments) == before: + continue + head, *tail = fragments + head.function.name = name + head.function.arguments = arguments + for fragment in tail: + fragment.function.arguments = "" + async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index b0f79552bc5..447870fc0be 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -101,6 +101,18 @@ if TYPE_CHECKING: from litellm.types.llms.openai import ResponseInputParam +class _ToolCallShape(NamedTuple): + name: str | None + arguments: str + + +def _tool_call_shapes(tool_calls: Sequence[ChatCompletionToolCallChunk]) -> tuple[_ToolCallShape, ...]: + return tuple( + _ToolCallShape(name=tool_call["function"].get("name"), arguments=tool_call["function"].get("arguments", "")) + for tool_call in tool_calls + ) + + class ResponseOutputEnvelope(TypedDict, total=False): """Dict form of a Responses API response, as far as guardrail write-back reads it.""" @@ -340,7 +352,7 @@ class OpenAIResponsesHandler(BaseTranslation): Methods can be overridden to customize behavior for different message formats. """ - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: """ @@ -754,6 +766,7 @@ class OpenAIResponsesHandler(BaseTranslation): if response_model: inputs["model"] = response_model + pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_to_check) guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, request_data=request_data, @@ -762,6 +775,12 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") + post_guardrail_tool_calls: Final = _tool_call_shapes( + returned_tool_calls + if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) + else tool_calls_to_check + ) # Write guardrailed texts back into the output items in-place. # final_chunk is a reference into responses_so_far so this @@ -784,6 +803,13 @@ class OpenAIResponsesHandler(BaseTranslation): stream_events=responses_so_far[:-1], rewrites_by_position=rewrites_by_position, ) + self._deliver_ended_stream_tool_call_rewrites( + responses_so_far=responses_so_far, + outputs=outputs, + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + post_guardrail_tool_calls=post_guardrail_tool_calls, + guardrail_name=guardrail_to_apply.guardrail_name or "unknown", + ) return responses_so_far # ------------------------------------------------------------------ # @@ -894,6 +920,89 @@ class OpenAIResponsesHandler(BaseTranslation): continue OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten) + def _deliver_ended_stream_tool_call_rewrites( + self, + responses_so_far: Sequence[object], + outputs: Sequence[object], + pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], + post_guardrail_tool_calls: tuple[_ToolCallShape, ...], + guardrail_name: str, + ) -> None: + """Write ended-stream guardrail tool-call rewrites into the completed + envelope's ``function_call`` items and sync the earlier stream events, + keyed by ``output_index``. The guardrail sees the envelope's function + calls in output order, which is how a rewritten call finds its item; a + rewrite whose calls do not line up with the envelope is reported as + undeliverable, so the pipeline executor discards it and releases the + original events.""" + if post_guardrail_tool_calls == pre_guardrail_tool_calls: + return + function_call_indices: Final = tuple( + output_idx + for output_idx, output_item in enumerate(outputs) + if stream_item_field(output_item, "type") == "function_call" + ) + if len(function_call_indices) != len(post_guardrail_tool_calls): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + raise UndeliverableStreamRewrite(guardrail_name) + rewrites_by_output_index: Final = MappingProxyType( + { + output_idx: after + for output_idx, before, after in zip( + function_call_indices, pre_guardrail_tool_calls, post_guardrail_tool_calls + ) + if after != before + } + ) + for output_idx, rewrite in rewrites_by_output_index.items(): + self._write_function_call_item(outputs[output_idx], rewrite.name, rewrite.arguments) + self._sync_stream_events_with_tool_call_rewrites( + stream_events=responses_so_far[:-1], + rewrites_by_output_index=rewrites_by_output_index, + ) + + def _sync_stream_events_with_tool_call_rewrites( + self, + stream_events: Sequence[object], + rewrites_by_output_index: Mapping[int, _ToolCallShape], + ) -> None: + """Sync pre-completion function-call events with the rewritten completed + response: the first ``function_call_arguments.delta`` for a rewritten call + carries the full rewritten arguments and the rest are blanked, while + ``function_call_arguments.done`` and ``output_item.done`` carry the full + rewritten arguments and ``output_item.added`` / ``output_item.done`` the + rewritten name, so every event a client may read agrees with the + rewritten ``response.completed`` payload.""" + delta_replacements: Final = MappingProxyType( + {index: chain((rewrite.arguments,), repeat("")) for index, rewrite in rewrites_by_output_index.items()} + ) + for event in stream_events: + output_index = stream_item_field(event, "output_index") + if not isinstance(output_index, int) or output_index not in rewrites_by_output_index: + continue + rewrite = rewrites_by_output_index[output_index] + match stream_item_field(event, "type"): + case "response.function_call_arguments.delta": + self._write_event_field(event, "delta", next(delta_replacements[output_index])) + case "response.function_call_arguments.done": + self._write_event_field(event, "arguments", rewrite.arguments) + case "response.output_item.added": + self._write_function_call_item(stream_item_field(event, "item"), rewrite.name, None) + case "response.output_item.done": + self._write_function_call_item(stream_item_field(event, "item"), rewrite.name, rewrite.arguments) + case _: + pass + + @staticmethod + def _write_function_call_item(item: object, name: str | None, arguments: str | None) -> None: + if not (isinstance(item, dict) or hasattr(item, "get")): + return + if name is not None: + OpenAIResponsesHandler._write_event_field(item, "name", name) + if arguments is not None: + OpenAIResponsesHandler._write_event_field(item, "arguments", arguments) + def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool: """ Check if the streaming has ended. diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 9bc10949e9f..3e112bf8e67 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -85,10 +85,10 @@ def _logged_by_inner_guardrail(method: _GuardrailMethodT) -> _GuardrailMethodT: class _StreamRewriteObserver(CustomGuardrail): """Stand-in handed to the endpoint translation in place of a streaming pipeline step's guardrail. It records whether the guardrail returned different output than it was given, - which for guardrails like Bedrock's ANONYMIZED action is only known at runtime. Text - rewrites are deliverable on translations that write them back across the buffered chunks - (``delivers_ended_stream_text_rewrites``); tool-call rewrites and text rewrites on any - other translation are discarded by the executor, which releases the original chunks. + which for guardrails like Bedrock's ANONYMIZED action is only known at runtime. Text and + tool-call rewrites are deliverable on translations that write them back across the + buffered chunks (``delivers_ended_stream_rewrites``); rewrites on any other translation + are discarded by the executor, which releases the original chunks. The inner guardrail's ``apply_guardrail`` already records the guardrail information and span, so the observer's stays out of ``log_guardrail_information``.""" @@ -290,13 +290,13 @@ class PipelineExecutor: litellm_logging_obj: "LiteLLMLoggingObj | None", ) -> None: """Run one streaming post_call step through the endpoint translation, delivering - text rewrites on translations that support ended-stream write-back. A rewrite that - cannot reach the client yet (a tool-call rewrite, a text rewrite on a translation - without write-back, or one the translation refused with - ``UndeliverableStreamRewrite``) is discarded: the buffered chunks go back to the - originals and the step passes, so the client gets the stream the merge base sent.""" + text and tool-call rewrites on translations that support ended-stream write-back. A + rewrite that cannot reach the client yet (one on a translation without write-back, or + one the translation refused with ``UndeliverableStreamRewrite``) is discarded: the + buffered chunks go back to the originals and the step passes, so the client gets the + stream the merge base sent.""" observer: Final = _StreamRewriteObserver(callback) - deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_text_rewrites + deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_rewrites originals: Final = copy.deepcopy(streaming_chunks) try: if deliver_rewrites: @@ -319,7 +319,7 @@ class PipelineExecutor: except UndeliverableStreamRewrite: _release_original_chunks(step.guardrail, streaming_chunks, originals) else: - if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites): + if not deliver_rewrites and (observer.rewrote_texts or observer.rewrote_tool_calls): _release_original_chunks(step.guardrail, streaming_chunks, originals) if not callback.records_own_guardrail_information: add_guardrail_to_applied_guardrails_header(request_data=hook_input, guardrail_name=step.guardrail) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8b36061c6be..76b9b4b44ed 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3445,11 +3445,11 @@ class ProxyLogging: assembled output through the endpoint guardrail translation, the same machinery flat post_call guardrails use at end of stream. An allow releases the buffered chunks: verbatim when no guardrail rewrote the - output, rewritten in place when one rewrote text and the translation - delivers ended-stream rewrites (later steps then re-scan the rewritten - chunks, so rewrites chain). A rewrite the translation cannot deliver - yet (a tool-call rewrite, or a text rewrite on a route without - write-back) is discarded by the executor and the original chunks are + output, rewritten in place when one rewrote text or a tool call and the + translation delivers ended-stream rewrites (later steps then re-scan the + rewritten chunks, so rewrites chain). A rewrite the translation cannot + deliver yet (one on a route without write-back, or a shape the route + refuses) is discarded by the executor and the original chunks are released, as is a buffered shape no translation resolves; a block or modify_response terminates with the translation's block chunks or the raised error. diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index bf40f781fa3..bd2dff9b33c 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -315,6 +315,72 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: assert "event: message_start" in raw and "event: message_stop" in raw assert '"stop_reason": "end_turn"' in raw + @staticmethod + def _ended_tool_use_sse_chunks() -> list: + events = [ + ("message_start", {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "claude-sonnet-4-5", "content": [], "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 0}}}), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup_fruit", "input": {}}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": ""}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"fruit": "persim'}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": 'mon"}'}}), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, "usage": {"output_tokens": 2}}), + ("message_stop", {"type": "message_stop"}), + ] + return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events] + + @staticmethod + def _argument_masking_guardrail() -> CustomGuardrail: + class MaskArguments(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + for tool_call in inputs.get("tool_calls", []): + tool_call.function.arguments = '{"fruit": "[MASKED]"}' + return inputs + + return MaskArguments(guardrail_name="test") + + @staticmethod + def _partial_jsons(chunks: list) -> list: + return [ + json.loads(line[len("data:") :].strip())["delta"]["partial_json"] + for chunk in chunks + for line in chunk.decode().split("\n") + if line.startswith("data:") and json.loads(line[len("data:") :].strip()).get("type") == "content_block_delta" + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_tool_use_input_back_into_sse_chunks(self): + handler = AnthropicMessagesHandler() + chunks = self._ended_tool_use_sse_chunks() + + result = await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + assert result is chunks + assert self._partial_jsons(chunks) == ['{"fruit": "[MASKED]"}', "", ""] + raw = b"".join(chunks).decode() + assert '"name": "lookup_fruit"' in raw and '"id": "toolu_1"' in raw + assert '"stop_reason": "tool_use"' in raw + assert "persim" not in raw + + @pytest.mark.asyncio + async def test_ended_stream_tool_use_rewrite_leaves_chunks_untouched_by_default(self): + handler = AnthropicMessagesHandler() + chunks = self._ended_tool_use_sse_chunks() + original = [bytes(chunk) for chunk in chunks] + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=MagicMock(), + ) + + assert chunks == original + @pytest.mark.asyncio async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self): handler = AnthropicMessagesHandler() diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index aff0530ee8b..be168ec83e6 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1113,6 +1113,79 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert chunks[1].choices[0].delta.content in (None, "") assert chunks[1].choices[0].finish_reason == "stop" + @staticmethod + def _ended_tool_call_stream_chunks() -> list: + from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponseStream, + StreamingChoices, + ) + + def chunk(tool_call: ChatCompletionDeltaToolCall | None, finish_reason: Optional[str] = None): + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="gpt-4", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta(tool_calls=[tool_call] if tool_call else None), + finish_reason=finish_reason, + ) + ], + ) + + def fragment(arguments: str, name: Optional[str] = None, call_id: Optional[str] = None): + return ChatCompletionDeltaToolCall( + id=call_id, index=0, type="function", function=Function(name=name, arguments=arguments) + ) + + return [ + chunk(fragment("", name="lookup_fruit", call_id="call_1")), + chunk(fragment('{"fruit":')), + chunk(fragment(' "persimmon"}')), + chunk(None, finish_reason="tool_calls"), + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_tool_call_arguments_back_into_chunks(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="test") + chunks = self._ended_tool_call_stream_chunks() + + result = await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert result is chunks + fragments = [chunk.choices[0].delta.tool_calls for chunk in chunks[:3]] + assert [fragment[0].function.arguments for fragment in fragments] == ['{"fruit": "PERSIMMON"}', "", ""] + assert fragments[0][0].function.name == "lookup_fruit" + assert fragments[0][0].id == "call_1" + assert chunks[3].choices[0].delta.tool_calls is None + assert chunks[3].choices[0].finish_reason == "tool_calls" + + @pytest.mark.asyncio + async def test_ended_stream_tool_call_rewrite_leaves_chunks_untouched_by_default(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="test") + chunks = self._ended_tool_call_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + ) + + fragments = [chunk.choices[0].delta.tool_calls for chunk in chunks[:3]] + assert [fragment[0].function.arguments for fragment in fragments] == ["", '{"fruit":', ' "persimmon"}'] + @pytest.mark.asyncio async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self): handler = OpenAIChatCompletionsHandler() diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 33c8c97fea7..a024fc7ff81 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -1195,6 +1195,94 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert events[4]["item"]["content"][0]["text"] == "hello [MASKED]" assert events[5]["response"]["output"][0]["content"][0]["text"] == "hello [MASKED]" + @staticmethod + def _ended_function_call_stream_events() -> List[dict]: + def item(arguments: str, status: str) -> dict: + return { + "type": "function_call", + "id": "fc_123", + "call_id": "call_123", + "name": "lookup_fruit", + "arguments": arguments, + "status": status, + } + + return [ + {"type": "response.output_item.added", "output_index": 0, "item": item("", "in_progress")}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_123", "output_index": 0, "delta": '{"fruit":'}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_123", "output_index": 0, "delta": ' "persimmon"}'}, + { + "type": "response.function_call_arguments.done", + "item_id": "fc_123", + "output_index": 0, + "arguments": '{"fruit": "persimmon"}', + }, + {"type": "response.output_item.done", "output_index": 0, "item": item('{"fruit": "persimmon"}', "completed")}, + { + "type": "response.completed", + "response": { + "id": "resp_123", + "model": "gpt-4o", + "output": [item('{"fruit": "persimmon"}', "completed")], + "status": "completed", + }, + }, + ] + + @staticmethod + def _argument_masking_guardrail() -> CustomGuardrail: + class MaskArguments(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + tool_calls = [ + {**tool_call, "function": {**tool_call["function"], "arguments": '{"fruit": "[MASKED]"}'}} + for tool_call in inputs.get("tool_calls", []) + ] + return {**inputs, "tool_calls": tool_calls} + + return MaskArguments(guardrail_name="test-mask-arguments") + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_function_call_events(self): + handler = OpenAIResponsesHandler() + events = self._ended_function_call_stream_events() + + result = await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert result is events + assert events[0]["item"]["arguments"] == "" + assert events[1]["delta"] == '{"fruit": "[MASKED]"}' + assert events[2]["delta"] == "" + assert events[3]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[4]["item"]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[5]["response"]["output"][0]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[5]["response"]["output"][0]["name"] == "lookup_fruit" + + @pytest.mark.asyncio + async def test_ended_stream_function_call_rewrite_leaves_events_untouched_by_default(self): + handler = OpenAIResponsesHandler() + events = self._ended_function_call_stream_events() + + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + ) + + assert events[1]["delta"] == '{"fruit":' + assert events[3]["arguments"] == '{"fruit": "persimmon"}' + assert events[5]["response"]["output"][0]["arguments"] == '{"fruit": "persimmon"}' + @pytest.mark.asyncio @pytest.mark.parametrize("terminal_type", ["response.incomplete", "response.failed"]) async def test_deliver_ended_stream_rewrites_syncs_non_completed_terminals(self, terminal_type): diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 54ef1f79f4d..ee75003db9f 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -924,7 +924,7 @@ class _TextReturningGuardrail(CustomGuardrail): class _TextTranslation: - delivers_ended_stream_text_rewrites = False + delivers_ended_stream_rewrites = False def __init__(self): self.seen_guardrail_names = [] @@ -946,7 +946,7 @@ class _WritingTranslation: """Writes the guardrail's text (and tool-call) outputs back into the buffered chunks the way the chat/Responses/Messages handlers do on an ended stream.""" - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True async def process_output_streaming_response( self, @@ -970,7 +970,7 @@ class _WritingTranslation: class _RefusingTranslation: - delivers_ended_stream_text_rewrites = True + delivers_ended_stream_rewrites = True async def process_output_streaming_response( self, @@ -1088,13 +1088,27 @@ async def test_streaming_step_delivers_text_rewrite_through_writing_translation( @pytest.mark.asyncio -async def test_streaming_step_discards_tool_call_rewrite_and_restores_written_text(monkeypatch, caplog): +async def test_streaming_step_delivers_tool_call_rewrite_through_writing_translation(monkeypatch, caplog): monkeypatch.setattr(litellm, "callbacks", [_TextAndToolCallRewritingGuardrail(rewrite_tool_call=True)]) chunks = [_chunk()] with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): result = await _run_streaming_step(_WritingTranslation(), chunks) + assert result.terminal_action == "allow" + assert chunks[0]["text"] == "hello [MASKED]" + assert chunks[0]["tool_call"]["function"]["arguments"] == '{"ssn": "[MASKED]"}' + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_discards_tool_call_rewrite_when_translation_lacks_write_back(monkeypatch, caplog): + monkeypatch.setattr(litellm, "callbacks", [_TextAndToolCallRewritingGuardrail(rewrite_tool_call=True)]) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_streaming_step(_TextTranslation(), chunks) + _assert_passed_with_discard_warning(result, caplog) assert chunks == [_chunk()] diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 73dd6746b29..d27f504ba6d 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -1836,7 +1836,7 @@ def _echoed_tool_call_dicts(arguments: str) -> List[Dict[str, Any]]: @pytest.mark.asyncio @pytest.mark.parametrize("on_fail, on_error", [("block", None), ("next", "next")]) -async def test_streaming_iterator_hook_pipeline_releases_originals_on_runtime_tool_call_rewrite( +async def test_streaming_iterator_hook_pipeline_delivers_runtime_tool_call_rewrite( proxy_logging, make_user_api_key_auth, monkeypatch, on_fail, on_error, caplog ): transform = lambda inputs: {"tool_calls": _echoed_tool_call_dicts('{"ssn": "[MASKED]"}')} # noqa: E731 @@ -1855,9 +1855,12 @@ async def test_streaming_iterator_hook_pipeline_releases_originals_on_runtime_to delivered.append(item) assert len(delivered) == 2 - assert delivered[0].choices[0].delta.tool_calls[0].function.arguments == '{"ssn": "123"}' + delivered_tool_call = delivered[0].choices[0].delta.tool_calls[0] + assert delivered_tool_call.function.arguments == '{"ssn": "[MASKED]"}' + assert delivered_tool_call.function.name == "lookup" + assert delivered_tool_call.id == "call_1" assert delivered[1].choices[0].finish_reason == "tool_calls" - assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog)) + assert not any("discarded" in message for message in _warnings(caplog)) @pytest.mark.asyncio From e65c56876f9aae1f94120212a2092592792f6538 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:42:51 -0700 Subject: [PATCH 101/411] fix(guardrails): write tool-call rewrites into typed Responses envelopes The response.completed envelope carries its function_call output items as SDK objects without a get shim, so the write-back skipped them and the envelope still showed the original arguments after every stream event had been rewritten. Write the item whenever one is present, and cover the typed event shape the live proxy carries in the handler test. --- .../guardrail_translation/handler.py | 2 +- ...test_openai_responses_guardrail_handler.py | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 447870fc0be..208ceb959e6 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -996,7 +996,7 @@ class OpenAIResponsesHandler(BaseTranslation): @staticmethod def _write_function_call_item(item: object, name: str | None, arguments: str | None) -> None: - if not (isinstance(item, dict) or hasattr(item, "get")): + if item is None: return if name is not None: OpenAIResponsesHandler._write_event_field(item, "name", name) diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index a024fc7ff81..bdf7f83757c 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -1222,6 +1222,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: "type": "response.completed", "response": { "id": "resp_123", + "created_at": 1, "model": "gpt-4o", "output": [item('{"fruit": "persimmon"}', "completed")], "status": "completed", @@ -1268,6 +1269,51 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert events[5]["response"]["output"][0]["arguments"] == '{"fruit": "[MASKED]"}' assert events[5]["response"]["output"][0]["name"] == "lookup_fruit" + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_typed_function_call_events(self): + from litellm.types.llms.openai import ( + FunctionCallArgumentsDeltaEvent, + FunctionCallArgumentsDoneEvent, + OutputItemAddedEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ) + + handler = OpenAIResponsesHandler() + typed_events: List[Any] = [ + model.model_validate(event) + for model, event in zip( + ( + OutputItemAddedEvent, + FunctionCallArgumentsDeltaEvent, + FunctionCallArgumentsDeltaEvent, + FunctionCallArgumentsDoneEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ), + self._ended_function_call_stream_events(), + ) + ] + completed_event = typed_events[5] + assert isinstance(completed_event, ResponseCompletedEvent) + assert isinstance(completed_event.response, ResponsesAPIResponse) + assert isinstance(completed_event.response.output[0], ResponseFunctionToolCall) + + await handler.process_output_streaming_response( + responses_so_far=typed_events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert typed_events[1].delta == '{"fruit": "[MASKED]"}' + assert typed_events[2].delta == "" + assert typed_events[3].arguments == '{"fruit": "[MASKED]"}' + assert typed_events[4].item.arguments == '{"fruit": "[MASKED]"}' + assert completed_event.response.output[0].arguments == '{"fruit": "[MASKED]"}' + assert completed_event.response.output[0].name == "lookup_fruit" + @pytest.mark.asyncio async def test_ended_stream_function_call_rewrite_leaves_events_untouched_by_default(self): handler = OpenAIResponsesHandler() From 89c3a8216be6c42bc9d327e312c05d62c99118b9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:54:43 -0700 Subject: [PATCH 102/411] fix(bedrock): sign requests off the event loop on every async path SigV4 signing resolves AWS credentials, and botocore refreshes expiring credentials inside that signing with a blocking HTTP call. Every async Bedrock path that still signed on the event loop (/v1/messages, Converse, count tokens, the agent-runtime and Comprehend Medical pass-throughs, async-invoke status polling, realtime, AgentCore, SQS, S3) now signs on a worker thread, so one Bedrock request no longer stalls the whole worker. Fixes #40165 --- .../providers/bedrock_agentcore/handler.py | 7 +- litellm/integrations/s3_v2.py | 4 +- litellm/integrations/sqs.py | 2 +- litellm/llms/bedrock/base_aws_llm.py | 19 ++++- litellm/llms/bedrock/chat/converse_handler.py | 7 +- litellm/llms/bedrock/count_tokens/handler.py | 9 ++- litellm/llms/bedrock/embed/embedding.py | 48 ++++++------ litellm/llms/bedrock/realtime/handler.py | 5 +- litellm/llms/custom_httpx/llm_http_handler.py | 25 ++++-- .../llm_passthrough_endpoints.py | 78 +++++++++++-------- .../chat/test_bedrock_converse_handler.py | 25 +++++- .../test_bedrock_count_tokens_handler.py | 50 ++++++++++++ .../llms/bedrock/event_loop_probe.py | 52 +++++++++++++ .../llms/bedrock/test_base_aws_llm.py | 24 ++++++ .../custom_httpx/test_llm_http_handler.py | 60 ++++++++++++++ .../test_llm_pass_through_endpoints.py | 27 ++++++- 16 files changed, 363 insertions(+), 79 deletions(-) create mode 100644 tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py create mode 100644 tests/test_litellm/llms/bedrock/event_loop_probe.py diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py index a4e6fa50901..e01e215b5f0 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py @@ -5,6 +5,7 @@ Sends JSON-RPC envelopes directly to AgentCore endpoints, bypassing the completion bridge that would otherwise strip the envelope. """ +import asyncio import json from collections.abc import AsyncIterator, Mapping from typing import Any, Final @@ -45,7 +46,8 @@ class BedrockAgentCoreA2AHandler: Returns: A2A JSON-RPC response dict from the AgentCore agent """ - url, headers, body = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + url, headers, body = await asyncio.to_thread( + BedrockAgentCoreA2ATransformation.get_url_and_signed_request, request_id=request_id, params=params, litellm_params=litellm_params, @@ -91,7 +93,8 @@ class BedrockAgentCoreA2AHandler: Yields: A2A streaming response events from the AgentCore agent """ - url, headers, body = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + url, headers, body = await asyncio.to_thread( + BedrockAgentCoreA2ATransformation.get_url_and_signed_request, request_id=request_id, params=params, litellm_params=litellm_params, diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 712ce41d09e..e3297cfd27c 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -366,7 +366,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): # Sign the request aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers) aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name) - S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request) + await asyncio.to_thread(S3SigV4Auth(credentials, "s3", aws_region_name).add_auth, aws_request) # Prepare the signed headers signed_headers: Final = dict(aws_request.headers.items()) @@ -597,7 +597,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): # Sign the request aws_request: Final = AWSRequest(method="GET", url=url, headers=headers) - S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request) + await asyncio.to_thread(S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth, aws_request) # Prepare the signed headers signed_headers: Final = dict(aws_request.headers.items()) diff --git a/litellm/integrations/sqs.py b/litellm/integrations/sqs.py index 2b4c8c9928d..31cc20f5995 100644 --- a/litellm/integrations/sqs.py +++ b/litellm/integrations/sqs.py @@ -295,7 +295,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): data=prepped.body, headers=prepped.headers, ) - SigV4Auth(credentials, "sqs", self.sqs_region_name).add_auth(aws_request) + await asyncio.to_thread(SigV4Auth(credentials, "sqs", self.sqs_region_name).add_auth, aws_request) signed_headers: Final = dict(aws_request.headers.items()) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index a197702921f..bbd4988b204 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -1,3 +1,4 @@ +import asyncio import base64 import hashlib import json @@ -7,7 +8,7 @@ import urllib.parse from collections.abc import Callable, Mapping from datetime import datetime from threading import Lock -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast, get_args, overload +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, ParamSpec, TypeVar, cast, get_args, overload import httpx from pydantic import BaseModel, ValidationError @@ -1668,3 +1669,19 @@ class BaseAWSLLM: request_headers_dict["Authorization"] = incoming_authorization return request_headers_dict, request.body + + +_SignParams = ParamSpec("_SignParams") +_SignedRequest = TypeVar("_SignedRequest") + + +async def sign_request_off_loop_if_aws( + provider_config: object, + sign_request: Callable[_SignParams, _SignedRequest], + /, + *args: _SignParams.args, + **kwargs: _SignParams.kwargs, # kwargs-ok: ParamSpec forwarding keeps the wrapped sign_request signature +) -> _SignedRequest: + if isinstance(provider_config, BaseAWSLLM): + return await asyncio.to_thread(sign_request, *args, **kwargs) + return sign_request(*args, **kwargs) diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 984ba371898..3e8e1226a55 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -1,3 +1,4 @@ +import asyncio import json from collections.abc import Mapping from types import MappingProxyType @@ -136,7 +137,8 @@ class BedrockConverseLLM(BaseAWSLLM): ) data: Final = json.dumps(request_data) - prepped: Final = self.get_request_headers( + prepped: Final = await asyncio.to_thread( + self.get_request_headers, credentials=credentials, aws_region_name=litellm_params.get("aws_region_name") or "us-west-2", extra_headers=headers, @@ -206,7 +208,8 @@ class BedrockConverseLLM(BaseAWSLLM): ) data: Final = json.dumps(request_data) - prepped: Final = self.get_request_headers( + prepped: Final = await asyncio.to_thread( + self.get_request_headers, credentials=credentials, aws_region_name=litellm_params.get("aws_region_name") or "us-west-2", extra_headers=headers, diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py index 1fb53f6ff0a..cae2315e744 100644 --- a/litellm/llms/bedrock/count_tokens/handler.py +++ b/litellm/llms/bedrock/count_tokens/handler.py @@ -4,6 +4,7 @@ AWS Bedrock CountTokens API handler. Simplified handler leveraging existing LiteLLM Bedrock infrastructure. """ +import asyncio from typing import Any, Final import httpx @@ -12,7 +13,7 @@ import litellm from litellm._logging import verbose_logger from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client class BedrockCountTokensHandler(BedrockCountTokensConfig): @@ -27,6 +28,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): request_data: dict[str, Any], litellm_params: dict[str, Any], resolved_model: str, + client: AsyncHTTPHandler | None = None, ) -> dict[str, Any]: """ Handle a CountTokens request using existing LiteLLM patterns. @@ -75,7 +77,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): # Extract api_key for bearer token auth if provided api_key: Final = litellm_params.get("api_key", None) headers: Final = {"Content-Type": "application/json"} - signed_headers, signed_body = self._sign_request( + signed_headers, signed_body = await asyncio.to_thread( + self._sign_request, service_name="bedrock", headers=headers, optional_params=litellm_params, @@ -85,7 +88,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): api_key=api_key, ) - async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.BEDROCK) + async_client: Final = client or get_async_httpx_client(llm_provider=litellm.LlmProviders.BEDROCK) response: Final = await async_client.post( endpoint_url, diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index d3725434498..e5f94e51e6d 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -2,10 +2,11 @@ Handles embedding calls to Bedrock's `/invoke` endpoint """ +import asyncio import copy import json import urllib.parse -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final, get_args, overload import httpx @@ -26,7 +27,7 @@ from litellm.types.llms.bedrock import ( ) from litellm.types.utils import EmbeddingResponse, LlmProviders -from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token +from ..base_aws_llm import AWSPreparedRequest, BaseAWSLLM, Credentials, bedrock_bearer_token from ..common_utils import BedrockError from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config @@ -41,6 +42,20 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +def _sign_get_request( + credentials: Credentials, url: str, headers: Mapping[str, str], aws_region_name: str +) -> AWSPreparedRequest: + try: + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + except ImportError: + raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") + + request: Final = AWSRequest(method="GET", url=url, data=None, headers=dict(headers)) + SigV4Auth(credentials, "bedrock", aws_region_name).add_auth(request) + return request.prepare() + + class BedrockEmbedding(BaseAWSLLM): @overload def _load_credentials( @@ -599,9 +614,6 @@ class BedrockEmbedding(BaseAWSLLM): dict: Status response from AWS Bedrock """ - # Get AWS credentials using the same method as other Bedrock methods - credentials, _ = self._load_credentials(kwargs) - # Get the runtime endpoint endpoint_url, _ = self.get_runtime_endpoint( api_base=None, @@ -618,27 +630,13 @@ class BedrockEmbedding(BaseAWSLLM): # Prepare headers for GET request headers: Final = {"Content-Type": "application/json"} - # Use AWSRequest directly for GET requests (get_request_headers hardcodes POST) - try: - from botocore.auth import SigV4Auth - from botocore.awsrequest import AWSRequest - except ImportError: - raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") + def sign_status_request() -> AWSPreparedRequest: + credentials, _ = self._load_credentials(kwargs) + return _sign_get_request( + credentials=credentials, url=status_url, headers=headers, aws_region_name=aws_region_name + ) - # Create AWSRequest with GET method and encoded URL - request: Final = AWSRequest( - method="GET", - url=status_url, - data=None, # GET request, no body - headers=headers, - ) - - # Sign the request - SigV4Auth will create canonical string from request URL - sigv4: Final = SigV4Auth(credentials, "bedrock", aws_region_name) - sigv4.add_auth(request) - - # Prepare the request - prepped: Final = request.prepare() + prepped: Final = await asyncio.to_thread(sign_status_request) # LOGGING if logging_obj is not None: diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 42fe8941443..792e3169a7a 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -149,7 +149,8 @@ class BedrockRealtime(BaseAWSLLM): verbose_proxy_logger.debug("Bedrock Realtime: Connecting to %s with model %s", endpoint_uri, model) - credentials: Final = self.get_credentials( + credentials: Final = await asyncio.to_thread( + self.get_credentials, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, @@ -169,7 +170,7 @@ class BedrockRealtime(BaseAWSLLM): "or configure credentials in the environment" ), ) - frozen_credentials: Final = credentials.get_frozen_credentials() + frozen_credentials: Final = await asyncio.to_thread(credentials.get_frozen_credentials) # Initialize Bedrock client with aws_sdk_bedrock_runtime config: Final = Config( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 2f561809940..48dc009d116 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -77,6 +77,7 @@ from litellm.llms.base_llm.vector_store_files.transformation import ( BaseVectorStoreFilesConfig, ) from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws from litellm.llms.custom_httpx.container_handler import raise_for_error_status from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -1961,7 +1962,9 @@ class BaseLLMHTTPHandler: api_key=api_key, ) - signed_headers, signed_json_body = provider_config.sign_request( + signed_headers, signed_json_body = await sign_request_off_loop_if_aws( + provider_config, + provider_config.sign_request, headers=headers, optional_params=optional_params, request_data=data, @@ -2062,7 +2065,9 @@ class BaseLLMHTTPHandler: max_attempts, ) provider_config.transform_anthropic_messages_request_on_http_error(e=e, request_data=request_body) - headers, signed_json_body = provider_config.sign_request( + headers, signed_json_body = await sign_request_off_loop_if_aws( + provider_config, + provider_config.sign_request, headers=headers, optional_params=optional_params_dict, request_data=request_body, @@ -2222,7 +2227,9 @@ class BaseLLMHTTPHandler: stream=stream, ) - headers, signed_json_body = anthropic_messages_provider_config.sign_request( + headers, signed_json_body = await sign_request_off_loop_if_aws( + anthropic_messages_provider_config, + anthropic_messages_provider_config.sign_request, headers=headers, optional_params=dict(litellm_params), # dynamic aws_* params are passed under litellm_params request_data=request_body, @@ -2898,7 +2905,9 @@ class BaseLLMHTTPHandler: fake_stream=fake_stream, ) - headers, signed_body = responses_api_provider_config.sign_request( + headers, signed_body = await sign_request_off_loop_if_aws( + responses_api_provider_config, + responses_api_provider_config.sign_request, headers=headers, optional_params=dict(litellm_params), request_data=data, @@ -4606,7 +4615,9 @@ class BaseLLMHTTPHandler: ) data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data) - headers, signed_body = responses_api_provider_config.sign_request( + headers, signed_body = await sign_request_off_loop_if_aws( + responses_api_provider_config, + responses_api_provider_config.sign_request, headers=headers, optional_params=dict(litellm_params), request_data=data, @@ -9833,7 +9844,9 @@ class BaseLLMHTTPHandler: ) all_optional_params: Final[dict[str, object]] = dict(litellm_params) all_optional_params.update(vector_store_search_optional_params or {}) - headers, signed_json_body = vector_store_provider_config.sign_request( + headers, signed_json_body = await sign_request_off_loop_if_aws( + vector_store_provider_config, + vector_store_provider_config.sign_request, headers=headers, optional_params=all_optional_params, request_data=request_body, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 2ea46b740a8..6eb7b5d8541 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -8,6 +8,7 @@ Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. from __future__ import annotations +import asyncio import hmac import inspect import json @@ -15,6 +16,7 @@ import os import re from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from dataclasses import dataclass +from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast @@ -84,6 +86,9 @@ from litellm.utils import ProviderConfigManager from .passthrough_endpoint_router import PassthroughEndpointRouter if TYPE_CHECKING: + from botocore.awsrequest import AWSPreparedRequest + from botocore.credentials import Credentials + from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig from litellm.router import Router @@ -1099,13 +1104,6 @@ async def bedrock_proxy_route( """ create_request_copy(request) - try: - from botocore.auth import SigV4Auth - from botocore.awsrequest import AWSRequest - from botocore.credentials import Credentials - except ImportError: - raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") - aws_region_name: Final = get_secret_str(secret_name="AWS_REGION_NAME") if not _is_bedrock_agent_runtime_route(endpoint=endpoint): return await bedrock_llm_proxy_route( @@ -1139,17 +1137,20 @@ async def bedrock_proxy_route( from litellm.llms.bedrock.chat import BedrockConverseLLM bedrock_llm: Final = BedrockConverseLLM() - credentials: Final[Credentials] = bedrock_llm.get_credentials() - sigv4: Final = SigV4Auth(credentials, "bedrock", aws_region_name) - headers: Final = {"Content-Type": "application/json"} # Assuming the body contains JSON data, parse it try: data: Final = await _json_request_body(request) except Exception as e: raise HTTPException(status_code=400, detail={"error": e}) - _request: Final = AWSRequest(method="POST", url=str(updated_url), data=json.dumps(data), headers=headers) - sigv4.add_auth(_request) - prepped: Final = _request.prepare() + prepped: Final = await asyncio.to_thread( + _sign_aws_json_post, + get_credentials=bedrock_llm.get_credentials, + service_name="bedrock", + aws_region_name=aws_region_name, + url=str(updated_url), + body=json.dumps(data), + headers=MappingProxyType({"Content-Type": "application/json"}), + ) ## check for streaming is_streaming_request = False @@ -1177,6 +1178,25 @@ async def bedrock_proxy_route( return received_value +def _sign_aws_json_post( + get_credentials: Callable[[], Credentials], + service_name: str, + aws_region_name: str | None, + url: str, + body: str, + headers: Mapping[str, str], +) -> AWSPreparedRequest: + try: + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + except ImportError: + raise ImportError(f"Missing boto3 to call {service_name}. Run 'pip install boto3'.") + + aws_request: Final = AWSRequest(method="POST", url=url, data=body, headers=dict(headers)) + SigV4Auth(get_credentials(), service_name, aws_region_name).add_auth(aws_request) + return aws_request.prepare() + + COMPREHEND_MEDICAL_TARGET_PREFIX: Final = "ComprehendMedical_20181030" @@ -1207,13 +1227,6 @@ async def comprehend_medical_proxy_route( [Docs](https://docs.litellm.ai/docs/pass_through/comprehend_medical) """ - try: - from botocore.auth import SigV4Auth - from botocore.awsrequest import AWSRequest - from botocore.credentials import Credentials - except ImportError: - raise ImportError("Missing boto3 to call comprehendmedical. Run 'pip install boto3'.") - from .llm_provider_handlers.comprehend_medical_passthrough_logging_handler import ( COMPREHEND_MEDICAL_SUPPORTED_OPERATIONS, ) @@ -1246,18 +1259,21 @@ async def comprehend_medical_proxy_route( from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM - credentials: Final[Credentials] = BaseAWSLLM().get_credentials(aws_region_name=aws_region_name) - sigv4: Final = SigV4Auth(credentials, "comprehendmedical", aws_region_name) - headers: Final = MappingProxyType( - { - "Content-Type": "application/x-amz-json-1.1", - "X-Amz-Target": f"{COMPREHEND_MEDICAL_TARGET_PREFIX}.{operation}", - } - ) target_url: Final = f"https://comprehendmedical.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/" - _request: Final = AWSRequest(method="POST", url=target_url, data=json.dumps(data), headers=headers) - sigv4.add_auth(_request) - prepped: Final = _request.prepare() + prepped: Final = await asyncio.to_thread( + _sign_aws_json_post, + get_credentials=partial(BaseAWSLLM().get_credentials, aws_region_name=aws_region_name), + service_name="comprehendmedical", + aws_region_name=aws_region_name, + url=target_url, + body=json.dumps(data), + headers=MappingProxyType( + { + "Content-Type": "application/x-amz-json-1.1", + "X-Amz-Target": f"{COMPREHEND_MEDICAL_TARGET_PREFIX}.{operation}", + } + ), + ) endpoint_func: Final = create_pass_through_route( endpoint=operation, diff --git a/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py index f34b8eb1fb9..8831fd9061c 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -6,6 +6,7 @@ extension, and AWS credential resolution is stubbed so nothing reaches STS. from __future__ import annotations +import asyncio from unittest.mock import MagicMock, patch import httpx @@ -16,6 +17,7 @@ from litellm.llms.bedrock.chat.converse_handler import BedrockConverseLLM from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.rust_bridge import chat_completions as bridge from litellm.types.utils import ModelResponse +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe RUST_RESPONSE = { "created": 1_700_000_000, @@ -308,7 +310,9 @@ CONVERSE_RESPONSE = { } -async def _drive_async_completion(*, skip_pre_call_logging: bool, logging_obj): +async def _drive_async_completion( + *, skip_pre_call_logging: bool, logging_obj, credentials: Credentials = RESOLVED_CREDENTIALS +): """Run the real `async_completion` with a stubbed transport.""" import httpx as _httpx @@ -335,7 +339,7 @@ async def _drive_async_completion(*, skip_pre_call_logging: bool, logging_obj): stream=None, optional_params={"maxTokens": 16}, litellm_params={"aws_region_name": "us-west-2"}, - credentials=RESOLVED_CREDENTIALS, + credentials=credentials, headers={}, client=client, skip_pre_call_logging=skip_pre_call_logging, @@ -357,6 +361,23 @@ async def test_async_completion_logs_pre_call_by_default(): assert logging_obj.pre_call.call_count == 1 +@pytest.mark.asyncio +async def test_async_completion_signs_off_the_event_loop(monkeypatch): + """Regression for issue #40165: botocore refreshes expiring credentials inside SigV4 signing with a + blocking HTTP call, so `async_completion` must sign on a worker thread to keep the loop serving.""" + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + probe = EventLoopProbe() + release = asyncio.create_task(probe.release_refresh_from_the_loop()) + + response = await _drive_async_completion( + skip_pre_call_logging=False, logging_obj=MagicMock(), credentials=probe.credentials() + ) + await release + + assert response.choices[0].message.content == "hi" + assert probe.served_during_refresh is True + + def _sync_client_returning_converse_response(): client = MagicMock() client.post.side_effect = lambda **_kwargs: httpx.Response( diff --git a/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py b/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py new file mode 100644 index 00000000000..31e65bb6e27 --- /dev/null +++ b/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py @@ -0,0 +1,50 @@ +import asyncio +from unittest.mock import AsyncMock + +import httpx +import pytest + +from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe + + +class _ProbedCountTokensHandler(BedrockCountTokensHandler): + def __init__(self, probe: EventLoopProbe) -> None: + super().__init__() + self._probe = probe + + def get_credentials(self, **kwargs): + return self._probe.credentials() + + +@pytest.mark.asyncio +async def test_handle_count_tokens_request_signs_off_the_event_loop(monkeypatch): + """Regression for issue #40165: the count_tokens handler signed on the loop, so botocore's blocking + credential refresh inside SigV4 stalled every other request on the worker.""" + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + probe = EventLoopProbe() + client = AsyncMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"inputTokens": 7}, + request=httpx.Request("POST", "https://bedrock-runtime.us-west-2.amazonaws.com/"), + ) + ) + release = asyncio.create_task(probe.release_refresh_from_the_loop()) + + result = await _ProbedCountTokensHandler(probe).handle_count_tokens_request( + request_data={ + "model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "messages": [{"role": "user", "content": "hi"}], + }, + litellm_params={"aws_region_name": "us-west-2"}, + resolved_model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + client=client, + ) + await release + + assert result == {"input_tokens": 7} + assert client.post.call_args.kwargs["headers"]["Authorization"].startswith("AWS4-HMAC-SHA256") + assert probe.served_during_refresh is True diff --git a/tests/test_litellm/llms/bedrock/event_loop_probe.py b/tests/test_litellm/llms/bedrock/event_loop_probe.py new file mode 100644 index 00000000000..68acce308b8 --- /dev/null +++ b/tests/test_litellm/llms/bedrock/event_loop_probe.py @@ -0,0 +1,52 @@ +"""Refreshable credentials whose refresh only completes while the event loop keeps serving.""" + +from __future__ import annotations + +import asyncio +import threading +from datetime import datetime, timedelta, timezone +from typing import Final + +from botocore.credentials import RefreshableCredentials + +REFRESH_RELEASE_TIMEOUT_SECONDS: Final = 2.0 + + +class EventLoopProbe: + """Blocks inside botocore's credential refresh until a coroutine on the loop releases it. + + Signing on the event loop thread can never be released, so `served_during_refresh` reads False there + and True only when the refresh ran on another thread while the loop stayed responsive. + """ + + def __init__(self) -> None: + self.refresh_started: Final = threading.Event() + self.loop_served: Final = threading.Event() + self.served_during_refresh: bool | None = None + + def refresh(self) -> dict[str, str | None]: + self.refresh_started.set() + served: Final = self.loop_served.wait(timeout=REFRESH_RELEASE_TIMEOUT_SECONDS) + if self.served_during_refresh is None: + self.served_during_refresh = served + return { + "access_key": "AKIAREFRESHED", + "secret_key": "refreshed-secret", + "token": None, + "expiry_time": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + } + + def credentials(self) -> RefreshableCredentials: + return RefreshableCredentials( + access_key="AKIASTALE", + secret_key="stale-secret", + token=None, + expiry_time=datetime.now(timezone.utc) + timedelta(seconds=60), + refresh_using=self.refresh, + method="event-loop-probe", + ) + + async def release_refresh_from_the_loop(self) -> None: + while not self.refresh_started.is_set(): + await asyncio.sleep(0.005) + self.loop_served.set() diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index f854d806bdc..b3fc8dbe959 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -1,3 +1,4 @@ +import asyncio import json import os import threading @@ -22,7 +23,9 @@ from litellm.llms.bedrock.base_aws_llm import ( AwsAuthError, BaseAWSLLM, Boto3CredentialsInfo, + sign_request_off_loop_if_aws, ) +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe # Global variable for the base_aws_llm.py file path @@ -3215,3 +3218,24 @@ class TestGetRequestHeadersResign: extra_headers={"Authorization": "Bearer foo"}, ) assert prepped.headers["Authorization"] == "Bearer foo" + + +@pytest.mark.asyncio +async def test_sign_request_off_loop_if_aws_keeps_the_loop_serving_while_credentials_refresh(): + """Regression for issue #40165: an AWS provider's signing (and the botocore credential refresh + inside it) must run off the event loop, so other requests keep being served meanwhile.""" + probe = EventLoopProbe() + + def sign(headers: dict[str, str]) -> dict[str, str]: + request = AWSRequest( + method="POST", url="https://bedrock-runtime.us-west-2.amazonaws.com/", data="{}", headers=headers + ) + SigV4Auth(probe.credentials(), "bedrock", "us-west-2").add_auth(request) + return dict(request.headers) + + release = asyncio.create_task(probe.release_refresh_from_the_loop()) + signed = await sign_request_off_loop_if_aws(BaseAWSLLM(), sign, headers={"Content-Type": "application/json"}) + await release + + assert "Authorization" in signed + assert probe.served_during_refresh is True diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index e16855da8cb..6799850c4ae 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -29,10 +29,14 @@ from litellm.llms.custom_httpx.llm_http_handler import ( _rust_responses_websocket_enabled, ) from litellm.llms.azure.videos.transformation import AzureVideoConfig +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeMessagesConfig, +) from litellm.llms.openai.videos.transformation import OpenAIVideoConfig from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ImageObject, ImageResponse, ModelResponse, TranscriptionResponse +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe _ACTIVE_KEY = "_code_interpreter_interception_active" _SANDBOX_KEY = "_code_interpreter_interception_sandbox_key" @@ -749,6 +753,62 @@ async def test_anthropic_messages_streaming_response_aclose_closes_agentic_upstr assert tracker.closed is True +class _ProbedBedrockMessagesConfig(AmazonAnthropicClaudeMessagesConfig): + def __init__(self, probe: EventLoopProbe) -> None: + super().__init__() + self._probe = probe + + def get_credentials(self, **kwargs): + return self._probe.credentials() + + +@pytest.mark.asyncio +async def test_async_anthropic_messages_handler_signs_bedrock_off_the_event_loop(monkeypatch): + """Regression for issue #40165: /v1/messages on Bedrock signed on the loop, so botocore's blocking + credential refresh inside SigV4 stalled every other request on the worker.""" + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + probe = EventLoopProbe() + handler = BaseLLMHTTPHandler() + upstream_response = httpx.Response( + 200, + json={ + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "hi"}], + "model": "claude-haiku-4-5-20251001", + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + request=httpx.Request("POST", "https://bedrock-runtime.us-west-2.amazonaws.com/"), + ) + mock_client = AsyncMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=upstream_response) + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + mock_logging_obj.dynamic_success_callbacks = None + release = asyncio.create_task(probe.release_refresh_from_the_loop()) + + await handler.async_anthropic_messages_handler( + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_provider_config=_ProbedBedrockMessagesConfig(probe), + anthropic_messages_optional_request_params={"max_tokens": 16}, + custom_llm_provider="bedrock", + litellm_params=GenericLiteLLMParams(aws_region_name="us-west-2"), + logging_obj=mock_logging_obj, + client=mock_client, + stream=False, + kwargs={}, + ) + await release + + sent_headers = mock_client.post.call_args.kwargs["headers"] + assert sent_headers["Authorization"].startswith("AWS4-HMAC-SHA256") + assert probe.served_during_refresh is True + + @pytest.mark.asyncio async def test_async_anthropic_messages_handler_passes_litellm_metadata(): """Ensure litellm_metadata from kwargs is forwarded via update_from_kwargs. diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index d9969dd1dc9..a1cfd973b75 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1,3 +1,4 @@ +import asyncio import base64 import contextlib import json @@ -19,6 +20,7 @@ from starlette.datastructures import FormData import litellm from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, @@ -1852,11 +1854,11 @@ class TestBedrockAgentRuntimePassthroughToggle: return request @contextlib.contextmanager - def _patched_dispatch(self, general_settings: Mapping[str, object]): + def _patched_dispatch(self, general_settings: Mapping[str, object], credentials: object | None = None): from botocore.credentials import Credentials bedrock_llm: Final = Mock() - bedrock_llm.get_credentials = Mock(return_value=Credentials("ak", "sk")) + bedrock_llm.get_credentials = Mock(return_value=credentials or Credentials("ak", "sk")) forwarder: Final = AsyncMock(return_value="forwarded") with ( @@ -1891,6 +1893,27 @@ class TestBedrockAgentRuntimePassthroughToggle: forwarder.assert_awaited_once() assert "bedrock-agent-runtime.us-east-1.amazonaws.com" in create_route.call_args.kwargs["target"] + @pytest.mark.asyncio + async def test_agent_runtime_dispatch_signs_off_the_event_loop(self, monkeypatch): + """Regression for issue #40165: the agent-runtime pass-through signed on the loop, so botocore's + blocking credential refresh inside SigV4 stalled every other request on the worker.""" + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + probe: Final = EventLoopProbe() + release: Final = asyncio.create_task(probe.release_refresh_from_the_loop()) + + with self._patched_dispatch(MappingProxyType({}), credentials=probe.credentials()) as (create_route, forwarder): + result: Final = await bedrock_proxy_route( + endpoint=self.AGENT_RUNTIME_ENDPOINT, + request=self._mock_request(), + fastapi_response=Mock(), + user_api_key_dict=UserAPIKeyAuth(), + ) + await release + + assert result == "forwarded" + assert create_route.call_args.kwargs["custom_headers"]["Authorization"].startswith("AWS4-HMAC-SHA256") + assert probe.served_during_refresh is True + @pytest.mark.asyncio @pytest.mark.parametrize("value", (True, "true", "True")) async def test_agent_runtime_dispatch_rejected_when_disabled(self, value: bool | str): From 3caa3b60d5a319efa78871198081783dd4c70ad1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:58:18 -0700 Subject: [PATCH 103/411] feat(guardrails): run post_call policy pipelines on background Responses retrieval A POST /v1/responses with background: true returns a queued response, so the post_call pipelines attached at submit time had nothing to inspect. They now defer on queued and in_progress responses and run on GET /v1/responses/{id} instead: the retrieval resolves the response id back to its deployment, re-attaches the policies that governed the original model, and reports them in the x-litellm-applied-* headers of the retrieval response. --- litellm/proxy/common_request_processing.py | 8 +- .../proxy/policy_engine/response_retrieval.py | 116 +++++++++++++ litellm/proxy/utils.py | 43 +++-- .../policy_engine/test_response_retrieval.py | 157 ++++++++++++++++++ .../proxy/test_common_request_processing.py | 109 ++++++++++++ .../proxy_logging/test_guardrail_pipeline.py | 95 +++++++---- 6 files changed, 485 insertions(+), 43 deletions(-) create mode 100644 litellm/proxy/policy_engine/response_retrieval.py create mode 100644 tests/test_litellm/proxy/policy_engine/test_response_retrieval.py diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9720e4b1cf8..1c1eda73c0c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -184,6 +184,7 @@ from litellm.proxy.litellm_pre_call_utils import ( refresh_proxy_server_request_body_snapshot, reject_url_valued_destination, ) +from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.utils import ( ModelResponse, ModelResponseStream, @@ -1883,7 +1884,6 @@ class ProxyBaseLLMRequestProcessing: data=self.data, user_api_key_dict=user_api_key_dict, ) - # Calculate request queue time after add_litellm_data_to_request # which sets arrival_time in proxy_server_request. Ends at start_time # (not a freshly captured time.time() here) so this window is exactly @@ -2031,6 +2031,12 @@ class ProxyBaseLLMRequestProcessing: data=self.data, call_type=route_type, ) + if route_type == "aget_responses": + attach_post_call_pipelines_to_retrieval( + data=self.data, + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + ) # Refresh AFTER pre_call_hook: guardrails (e.g. Presidio PII masking) may # have mutated `self.data` in place, and the audit-trail snapshot taken in diff --git a/litellm/proxy/policy_engine/response_retrieval.py b/litellm/proxy/policy_engine/response_retrieval.py new file mode 100644 index 00000000000..54b9acf6150 --- /dev/null +++ b/litellm/proxy/policy_engine/response_retrieval.py @@ -0,0 +1,116 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias + +from pydantic import TypeAdapter + +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket +from litellm.proxy.common_utils.callback_utils import ( + add_guardrail_to_applied_guardrails_header, + add_policy_sources_to_metadata, + add_policy_to_applied_policies_header, +) +from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body +from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry +from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher +from litellm.proxy.policy_engine.policy_registry import get_policy_registry +from litellm.proxy.policy_engine.policy_resolver import PolicyResolver +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.proxy.policy_engine import PolicyMatchContext +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline + +if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.router import Router + +PolicyPipelines: TypeAlias = tuple[tuple[str, GuardrailPipeline], ...] + +_POLICY_PIPELINES_ADAPTER: Final = TypeAdapter(PolicyPipelines) + + +def _model_group_for_response_id(response_id: object, llm_router: "Router | None") -> str | None: + if llm_router is None or not isinstance(response_id, str): + return None + model_id: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id) + if model_id is None: + return None + deployment: Final = llm_router.get_deployment(model_id) + return deployment.model_name if deployment is not None else None + + +def _retrieval_context( + data: Mapping[str, object], user_api_key_dict: "UserAPIKeyAuth", model_group: str +) -> PolicyMatchContext: + team_alias: Final = user_api_key_dict.team_alias + key_alias: Final = user_api_key_dict.key_alias + return PolicyMatchContext( + team_alias=team_alias if isinstance(team_alias, str) else None, + key_alias=key_alias if isinstance(key_alias, str) else None, + model=model_group, + tags=get_tags_from_request_body(data) or None, + ) + + +def _post_call_pipelines_for_context(context: PolicyMatchContext) -> tuple[PolicyPipelines, Mapping[str, str]]: + matches: Final = get_attachment_registry().get_attached_policies_with_reasons(context) + if not matches: + return (), MappingProxyType({}) + applied_policy_names: Final = PolicyMatcher.get_policies_with_matching_conditions( + policy_names=[match["policy_name"] for match in matches], # mutable-ok: the matcher takes a list + context=context, + ) + post_call_pipelines: Final = tuple( + (policy_name, pipeline) + for policy_name, pipeline in PolicyResolver.resolve_pipelines_for_context( + context=context, policy_names=applied_policy_names + ) + if pipeline.mode == "post_call" + ) + return post_call_pipelines, MappingProxyType({match["policy_name"]: match["matched_via"] for match in matches}) + + +def attach_post_call_pipelines_to_retrieval( + data: dict, # mutable-ok: the proxy's request-state dict, written in place like every other policy engine hook + user_api_key_dict: "UserAPIKeyAuth", + llm_router: "Router | None", +) -> None: + if not get_policy_registry().is_initialized(): + return + model_group: Final = _model_group_for_response_id(data.get("response_id"), llm_router) + if model_group is None: + return + context: Final = _retrieval_context(data, user_api_key_dict, model_group) + post_call_pipelines, policy_sources = _post_call_pipelines_for_context(context) + _, bucket = get_or_create_metadata_bucket(data) + already_attached: Final = _POLICY_PIPELINES_ADAPTER.validate_python(bucket.get("_guardrail_pipelines") or ()) + attached_policy_names: Final = frozenset(policy_name for policy_name, _pipeline in already_attached) + added: Final = tuple( + (policy_name, pipeline) + for policy_name, pipeline in post_call_pipelines + if policy_name not in attached_policy_names + ) + if not added: + return + pipelines: Final = (*already_attached, *added) + bucket["_guardrail_pipelines"] = pipelines + bucket["_pipeline_managed_guardrails"] = frozenset( + step.guardrail for _policy_name, pipeline in pipelines for step in pipeline.steps + ) + for policy_name, _pipeline in added: + add_policy_to_applied_policies_header(request_data=data, policy_name=policy_name) + for _policy_name, pipeline in added: + for step in pipeline.steps: + add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=step.guardrail) + add_policy_sources_to_metadata( + request_data=data, + policy_sources={ # mutable-ok: add_policy_sources_to_metadata takes a dict + policy_name: policy_sources[policy_name] for policy_name, _pipeline in added + }, + ) + verbose_proxy_logger.debug( + "Policy engine: attached post_call pipelines to the retrieval of background response %s (model group %s): %s", + data.get("response_id"), + model_group, + ", ".join(policy_name for policy_name, _pipeline in added), + ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8b36061c6be..3d03d3858ee 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -171,6 +171,7 @@ from litellm.repositories.verification_token_repository import ( ) from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.mcp import ( MCPDuringCallResponseObject, MCPPreCallRequestObject, @@ -525,15 +526,21 @@ def _post_call_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "Guardr ) -def _warn_background_skips_post_call_pipelines(data: Mapping[str, object]) -> None: - if data.get("background") is not True: - return +_PENDING_BACKGROUND_RESPONSE_STATUSES: Final = frozenset(("queued", "in_progress")) + + +def _is_pending_background_response(response: LLMResponseTypes) -> bool: + return isinstance(response, ResponsesAPIResponse) and response.status in _PENDING_BACKGROUND_RESPONSE_STATUSES + + +def _log_deferred_post_call_pipelines(data: Mapping[str, object], response: ResponsesAPIResponse) -> None: policy_names: Final = tuple(policy_name for policy_name, _pipeline in _post_call_pipelines(data)) if not policy_names: return - verbose_proxy_logger.warning( - "Policies with post_call guardrail pipelines do not run on background responses yet; " - "the response is released ungoverned by them: %s", + verbose_proxy_logger.debug( + "Post_call guardrail pipelines wait for background response %s (status=%s) to be retrieved complete: %s", + response.id, + response.status, ", ".join(policy_names), ) @@ -1956,8 +1963,6 @@ class ProxyLogging: ) try: - _warn_background_skips_post_call_pipelines(data) - # Execute guardrail pipelines before the normal callback loop data, _ = await self._maybe_execute_pipelines( # rebind-ok: pipeline edits feed the callback loop below data=data, @@ -2927,6 +2932,24 @@ class ProxyLogging: daemon=True, ).start() + async def _run_post_call_pipelines( + self, + data: dict, # mutable-ok: same request-payload shape as post_call_success_hook's data + user_api_key_dict: UserAPIKeyAuth, + response: LLMResponseTypes, + ) -> LLMResponseTypes | None: + if _is_pending_background_response(response): + _log_deferred_post_call_pipelines(data, response) + return None + _, pipeline_response = await self._maybe_execute_pipelines( + data=data, + user_api_key_dict=user_api_key_dict, + call_type=getattr(data.get("litellm_logging_obj"), "call_type", None) or "acompletion", + event_hook="post_call", + response=response, + ) + return pipeline_response + async def post_call_success_hook( self, data: dict, @@ -2946,11 +2969,9 @@ class ProxyLogging: from litellm.proxy.proxy_server import llm_router from litellm.types.guardrails import GuardrailEventHooks - _, pipeline_response = await self._maybe_execute_pipelines( + pipeline_response: Final = await self._run_post_call_pipelines( data=data, user_api_key_dict=user_api_key_dict, - call_type=getattr(data.get("litellm_logging_obj"), "call_type", None) or "acompletion", - event_hook="post_call", response=response, ) if pipeline_response is not None: diff --git a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py new file mode 100644 index 00000000000..f7f29d6b46e --- /dev/null +++ b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py @@ -0,0 +1,157 @@ +import pytest + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry +from litellm.proxy.policy_engine.policy_registry import get_policy_registry +from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.router import Deployment, LiteLLM_Params + +GOVERNED_MODEL_GROUP = "gpt-5.4-mini" +GOVERNED_MODEL_ID = "deployment-governed" +UNGOVERNED_MODEL_GROUP = "gpt-4.1-mini" +UNGOVERNED_MODEL_ID = "deployment-ungoverned" + + +class FakeRouter: + def __init__(self, deployments: dict[str, Deployment]): + self._deployments = deployments + + def get_deployment(self, model_id: str) -> Deployment | None: + return self._deployments.get(model_id) + + +def _deployment(model_group: str, model_id: str) -> Deployment: + return Deployment( + model_name=model_group, + litellm_params=LiteLLM_Params(model=f"openai/{model_group}"), + model_info={"id": model_id}, + ) + + +def _router() -> FakeRouter: + return FakeRouter( + { + GOVERNED_MODEL_ID: _deployment(GOVERNED_MODEL_GROUP, GOVERNED_MODEL_ID), + UNGOVERNED_MODEL_ID: _deployment(UNGOVERNED_MODEL_GROUP, UNGOVERNED_MODEL_ID), + } + ) + + +def _encoded_response_id(model_id: str) -> str: + return ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id=model_id, response_id="resp_upstream" + ) + + +def _pipeline_policy(guardrail: str, mode: str = "post_call") -> dict[str, object]: + return { + "guardrails": {"add": [guardrail]}, + "pipeline": {"mode": mode, "steps": [{"guardrail": guardrail, "on_pass": "allow", "on_fail": "block"}]}, + } + + +@pytest.fixture +def policy_engine(): + policy_registry = get_policy_registry() + attachment_registry = get_attachment_registry() + policy_registry.load_policies( + { + "response-governance": _pipeline_policy("output-word-filter"), + "input-governance": _pipeline_policy("input-word-filter", mode="pre_call"), + "team-governance": _pipeline_policy("team-word-filter"), + } + ) + attachment_registry.load_attachments( + [ + {"policy": "response-governance", "models": [GOVERNED_MODEL_GROUP]}, + {"policy": "input-governance", "models": [GOVERNED_MODEL_GROUP]}, + {"policy": "team-governance", "teams": ["governed-team"]}, + ] + ) + yield + policy_registry.clear() + attachment_registry.clear() + + +def _retrieval_data(model_id: str) -> dict: + return {"response_id": _encoded_response_id(model_id), "litellm_metadata": {}} + + +def _attached_pipelines(data: dict) -> tuple[tuple[str, str], ...]: + return tuple( + (policy_name, ",".join(step.guardrail for step in pipeline.steps)) + for policy_name, pipeline in data["litellm_metadata"]["_guardrail_pipelines"] + ) + + +def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine): + data = _retrieval_data(GOVERNED_MODEL_ID) + + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + + assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) + assert data["litellm_metadata"]["_pipeline_managed_guardrails"] == frozenset({"output-word-filter"}) + assert data["litellm_metadata"]["applied_policies"] == ["response-governance"] + assert data["litellm_metadata"]["applied_guardrails"] == ["output-word-filter"] + assert data["litellm_metadata"]["policy_sources"] == {"response-governance": "model:gpt-5.4-mini"} + assert "model" not in data + assert "guardrails" not in data["litellm_metadata"] + + +def test_key_and_team_context_also_governs_retrieval(policy_engine): + data = _retrieval_data(UNGOVERNED_MODEL_ID) + + attach_post_call_pipelines_to_retrieval( + data=data, user_api_key_dict=UserAPIKeyAuth(team_alias="governed-team"), llm_router=_router() + ) + + assert _attached_pipelines(data) == (("team-governance", "team-word-filter"),) + + +def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine): + data = _retrieval_data(UNGOVERNED_MODEL_ID) + + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + + assert data == _retrieval_data(UNGOVERNED_MODEL_ID) + + +def test_already_attached_policy_is_not_attached_twice(policy_engine): + data = _retrieval_data(GOVERNED_MODEL_ID) + router = _router() + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) + + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) + + assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) + assert data["litellm_metadata"]["applied_policies"] == ["response-governance"] + + +@pytest.mark.parametrize( + "response_id", + ["resp_plain_upstream_id", _encoded_response_id("deployment-missing-from-router"), None], +) +def test_unresolvable_response_id_attaches_nothing(policy_engine, response_id): + data = {"response_id": response_id, "litellm_metadata": {}} + + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + + assert data == {"response_id": response_id, "litellm_metadata": {}} + + +def test_without_a_router_attaches_nothing(policy_engine): + data = _retrieval_data(GOVERNED_MODEL_ID) + + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) + + assert data == _retrieval_data(GOVERNED_MODEL_ID) + + +def test_without_policy_engine_attaches_nothing(): + get_policy_registry().clear() + data = _retrieval_data(GOVERNED_MODEL_ID) + + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + + assert data == _retrieval_data(GOVERNED_MODEL_ID) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 6acd9d7258e..9283c1ed865 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -8259,3 +8259,112 @@ class TestPassthroughHeadersAcceptImmutableMappings: assert merged["content-type"] == "text/event-stream" # the excluded hop-by-hop header is still dropped assert "transfer-encoding" not in merged + + +class TestBackgroundResponseRetrievalGovernance: + """LIT-7175: retrieving a background Response attaches the model's post_call policy pipelines.""" + + GOVERNED_MODEL_GROUP = "gpt-5.4-mini" + GOVERNED_MODEL_ID = "deployment-governed" + + @pytest.fixture + def policy_engine(self): + from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry + from litellm.proxy.policy_engine.policy_registry import get_policy_registry + + get_policy_registry().load_policies( + { + "response-governance": { + "guardrails": {"add": ["output-word-filter"]}, + "pipeline": { + "mode": "post_call", + "steps": [{"guardrail": "output-word-filter", "on_pass": "allow", "on_fail": "block"}], + }, + } + } + ) + get_attachment_registry().load_attachments( + [{"policy": "response-governance", "models": [self.GOVERNED_MODEL_GROUP]}] + ) + yield + get_policy_registry().clear() + get_attachment_registry().clear() + + def _router(self) -> MagicMock: + from litellm.types.router import Deployment, LiteLLM_Params + + router = MagicMock() + router.get_deployment.side_effect = lambda model_id: ( + Deployment( + model_name=self.GOVERNED_MODEL_GROUP, + litellm_params=LiteLLM_Params(model=f"openai/{self.GOVERNED_MODEL_GROUP}"), + model_info={"id": model_id}, + ) + if model_id == self.GOVERNED_MODEL_ID + else None + ) + return router + + async def _pre_call(self, route_type: str, monkeypatch) -> dict: + from litellm.responses.utils import ResponsesAPIRequestUtils + + client_facing_response_id = "resp_opaque-client-facing-id" + encoded_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id=self.GOVERNED_MODEL_ID, response_id="resp_upstream" + ) + processing_obj = ProxyBaseLLMRequestProcessing( + data={"response_id": client_facing_response_id, "litellm_metadata": {}} + ) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def passthrough_add_litellm_data_to_request(data, **kwargs): + return data + + async def decrypting_pre_call_hook(user_api_key_dict, data, call_type): + if data.get("response_id") == client_facing_response_id: + data["response_id"] = encoded_response_id + return data + + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + passthrough_add_litellm_data_to_request, + ) + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=decrypting_pre_call_hook) + proxy_config = MagicMock(spec=ProxyConfig) + proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(), + proxy_logging_obj=proxy_logging_obj, + proxy_config=proxy_config, + route_type=route_type, + llm_router=self._router(), + ) + return returned_data + + @pytest.mark.asyncio + async def test_retrieving_a_background_response_attaches_its_model_post_call_pipeline( + self, policy_engine, monkeypatch + ): + data = await self._pre_call("aget_responses", monkeypatch) + + assert data["response_id"].startswith("resp_bGl0ZWxsbTpjdXN0b21f") + pipelines = data["litellm_metadata"]["_guardrail_pipelines"] + assert [(policy_name, [step.guardrail for step in pipeline.steps]) for policy_name, pipeline in pipelines] == [ + ("response-governance", ["output-word-filter"]) + ] + assert data["litellm_metadata"]["applied_policies"] == ["response-governance"] + assert data["model"] is None + + @pytest.mark.asyncio + async def test_submitting_a_response_does_not_attach_pipelines_from_its_response_id( + self, policy_engine, monkeypatch + ): + data = await self._pre_call("aresponses", monkeypatch) + + assert "_guardrail_pipelines" not in data["litellm_metadata"] + assert "applied_policies" not in data["litellm_metadata"] diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 73dd6746b29..7d2c232ad97 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -30,6 +30,7 @@ from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_g from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.proxy.policy_engine.pipeline_types import ( GuardrailPipeline, PipelineStep, @@ -1419,8 +1420,69 @@ async def test_streaming_request_whose_pipeline_guardrail_is_missing_streams_ver assert any("response-governance" in message and "gr-post" in message for message in _warnings(caplog)) +def _background_response(status: str, text: str = "") -> ResponsesAPIResponse: + output = ( + [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]}] if text else [] + ) + return ResponsesAPIResponse(id="resp_bg", created_at=0, output=output, status=status) + + +def _output_blocking_callbacks(seen: dict[str, object]) -> list[CustomGuardrail]: + class OutputBlockingGuardrail(CustomGuardrail): + async def async_post_call_success_hook(self, data, user_api_key_dict, response): + seen["response"] = response + raise HTTPException(status_code=400, detail={"error": "output blocked"}) + + return [OutputBlockingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)] + + @pytest.mark.asyncio -async def test_pre_call_hook_accepts_background_request_with_post_call_pipeline( +@pytest.mark.parametrize("pending_status", ["queued", "in_progress"]) +async def test_post_call_success_hook_waits_for_pending_background_response_before_running_pipeline( + proxy_logging, make_user_api_key_auth, monkeypatch, caplog, pending_status +): + seen: dict[str, object] = {} + monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks(seen)) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(background=True) + response = _background_response(pending_status) + + with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): + out = await proxy_logging.post_call_success_hook( + data=data, response=response, user_api_key_dict=make_user_api_key_auth() + ) + + assert out is response + assert "response" not in seen + assert not _warnings(caplog) + assert any( + "response-governance" in record.getMessage() and pending_status in record.getMessage() + for record in caplog.records + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("final_status", ["completed", "incomplete"]) +async def test_post_call_success_hook_runs_pipeline_on_retrieved_background_response( + proxy_logging, make_user_api_key_auth, monkeypatch, final_status +): + seen: dict[str, object] = {} + monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks(seen)) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data() + response = _background_response(final_status, text="kumquat") + + with pytest.raises(HTTPException) as info: + await proxy_logging.post_call_success_hook( + data=data, response=response, user_api_key_dict=make_user_api_key_auth() + ) + + assert info.value.detail["error"] == "output blocked" + assert seen["response"] is response + + +@pytest.mark.asyncio +async def test_pre_call_hook_stays_quiet_on_background_request_with_post_call_pipeline( proxy_logging, make_user_api_key_auth, monkeypatch, caplog ): monkeypatch.setattr(litellm, "callbacks", []) @@ -1436,36 +1498,7 @@ async def test_pre_call_hook_accepts_background_request_with_post_call_pipeline( assert out is not None assert out.get("background") is True - assert any("response-governance" in message and "background" in message for message in _warnings(caplog)) - - -@pytest.mark.asyncio -async def test_pre_call_hook_stays_quiet_on_background_request_without_post_call_pipeline( - proxy_logging, make_user_api_key_auth, monkeypatch, caplog -): - seen: Dict[str, Any] = {} - monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)]) - pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")]) - data = { - "model": "m", - "messages": [{"role": "user", "content": "hi"}], - "background": True, - "metadata": { - "_guardrail_pipelines": [("request-governance", pre_call)], - "_pipeline_managed_guardrails": {"gr-post"}, - }, - } - - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - out = await proxy_logging.pre_call_hook( - user_api_key_dict=make_user_api_key_auth(), - data=data, - call_type="aresponses", - guardrails_only=True, - ) - - assert out is not None - assert not any("background" in message for message in _warnings(caplog)) + assert not _warnings(caplog) # --------------------------------------------------------------------------- From 2285640eeab7206f609d54286df0bac4130987f7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:02:39 -0700 Subject: [PATCH 104/411] fix(spend-tracking): recover key alias for session tokens from spend logs CLI session tokens are in-memory only and never get a LiteLLM_VerificationToken row, so the usage APIs could not resolve key_alias, team_id, or user_email for their spend rows: the exact join and the reverse-hash recovery both miss. The owner is written to LiteLLM_SpendLogs.metadata at request time under the same hashed api_key, so read it back from there for keys still unresolved after the token-table passes. The lookup is sha256-gated like the existing reverse-hash recovery and bounded to the records' startTime window (min date minus one day, max date plus two) so it stays on the startTime index. No migration. Also guard the window parser against the date=None rollup rows GROUPING SETS aggregation emits, which raised TypeError from strptime and turned the aggregated usage endpoints into HTTP 500s. --- .../common_daily_activity.py | 51 ++++++++-- .../spend_tracking/key_metadata_recovery.py | 48 ++++++++++ .../test_common_daily_activity.py | 53 +++++++++++ .../test_key_metadata_recovery.py | 95 ++++++++++++++++++- 4 files changed, 238 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index ce6a97708ab..c6c5aad5926 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -14,6 +14,7 @@ from litellm.proxy._types import CommonProxyErrors from litellm.proxy.spend_tracking.key_metadata_recovery import ( attach_user_emails, recover_double_hashed_key_metadata, + recover_key_metadata_from_spend_logs, ) from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled from litellm.proxy.utils import PrismaClient @@ -433,15 +434,37 @@ def update_breakdown_metrics( return breakdown +def _spend_logs_window(dates: AbstractSet[str | None]) -> tuple[datetime, datetime] | None: + parsed: Final = sorted(day for day in (_parse_spend_date(raw) for raw in dates) if day is not None) + if not parsed: + return None + return (parsed[0] - timedelta(days=1), parsed[-1] + timedelta(days=2)) + + +def _parse_spend_date(raw: str | None) -> datetime | None: + if not isinstance(raw, str): + return None + try: + return datetime.fromisoformat(raw) + except ValueError: + return None + + +_EMPTY_KEY_METADATA: Final[Mapping[str, _KeyMetadataDict]] = MappingProxyType({}) + + async def get_api_key_metadata( prisma_client: PrismaClient, api_keys: AbstractSet[str], + spend_logs_window: tuple[datetime, datetime] | None = None, ) -> Mapping[str, _KeyMetadataDict]: """Get api key metadata, falling back to deleted keys table for keys not found in active table. This ensures that key_alias and team_id are preserved in historical activity logs even after a key is deleted or regenerated. Also recovers aliases for api_key - values that were double-hashed by the v1.99 spend-log provenance gate. + values that were double-hashed by the v1.99 spend-log provenance gate, and, when + spend_logs_window is given, for keys never written to either token table (CLI + session tokens) from the spend-log rows those requests wrote in that window. """ key_records: Sequence[PrismaVerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many( where={"token": {"in": list(api_keys)}} @@ -481,11 +504,17 @@ async def get_api_key_metadata( ) still_missing: Final = api_keys - frozenset(result) - combined: Final = ( - result - if not still_missing - else MappingProxyType({**result, **(await recover_double_hashed_key_metadata(prisma_client, still_missing))}) + from_reverse_hash: Final = ( + await recover_double_hashed_key_metadata(prisma_client, still_missing) if still_missing else _EMPTY_KEY_METADATA ) + after_token_recovery: Final = MappingProxyType({**result, **from_reverse_hash}) + unresolved: Final = api_keys - frozenset(after_token_recovery) + from_spend_logs: Final = ( + await recover_key_metadata_from_spend_logs(prisma_client, unresolved, spend_logs_window) + if unresolved and spend_logs_window is not None + else _EMPTY_KEY_METADATA + ) + combined: Final = MappingProxyType({**after_token_recovery, **from_spend_logs}) return await attach_user_emails(prisma_client, combined) @@ -898,7 +927,9 @@ async def _aggregate_spend_records( api_key_metadata: dict[str, _KeyMetadataDict] = {} if api_keys: - api_key_metadata = await get_api_key_metadata(prisma_client, api_keys) + api_key_metadata = await get_api_key_metadata( + prisma_client, api_keys, _spend_logs_window(frozenset(record.date for record in records)) + ) return await asyncio.to_thread( _aggregate_spend_records_sync, @@ -1094,7 +1125,9 @@ async def _aggregate_grouping_sets_records( api_key_metadata: dict[str, _KeyMetadataDict] = {} if api_keys: - api_key_metadata = await get_api_key_metadata(prisma_client, api_keys) + api_key_metadata = await get_api_key_metadata( + prisma_client, api_keys, _spend_logs_window(frozenset(r.date for r in records)) + ) return await asyncio.to_thread( _aggregate_grouping_sets_records_sync, @@ -1357,7 +1390,9 @@ async def get_daily_activity_aggregated( r.api_key for r in entity_records if r.api_key and r.api_key != PTU_SENTINEL_API_KEY ) entity_key_metadata: Final = ( - await get_api_key_metadata(prisma_client, entity_api_keys) + await get_api_key_metadata( + prisma_client, entity_api_keys, _spend_logs_window(frozenset(r.date for r in entity_records)) + ) if entity_api_keys else {} # mutable-ok: matches the helper's dict return ) diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 7de18521edd..81c1f4cf119 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -1,5 +1,6 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet +from datetime import datetime from types import MappingProxyType from typing import Final, TypeVar @@ -27,6 +28,19 @@ WHERE encode(sha256(convert_to(token, 'UTF8')), 'hex') = ANY($1::text[]) ORDER BY token, deleted_at DESC """ +_SPEND_LOG_ALIAS_SQL: Final = """ +SELECT DISTINCT ON (api_key) + api_key AS digest, + metadata->>'user_api_key_alias' AS key_alias, + COALESCE(NULLIF(team_id, ''), metadata->>'user_api_key_team_id') AS team_id, + COALESCE(NULLIF("user", ''), metadata->>'user_api_key_user_id') AS user_id +FROM "LiteLLM_SpendLogs" +WHERE api_key = ANY($1::text[]) + AND "startTime" >= $2::timestamp + AND "startTime" < $3::timestamp +ORDER BY api_key, (metadata->>'user_api_key_alias') IS NULL, "startTime" DESC +""" + class KeyMetadataDict(TypedDict, total=False): key_alias: ReadOnly[str | None] @@ -168,6 +182,40 @@ async def recover_double_hashed_key_metadata( return MappingProxyType({**from_active, **from_deleted}) +async def recover_key_metadata_from_spend_logs( + prisma_client: PrismaClient, + missing_keys: AbstractSet[str], + window: tuple[datetime, datetime], +) -> Mapping[str, KeyMetadataDict]: + """ + Recover key_alias/team_id/user_id for hashed api_key values absent from both + verification-token tables, e.g. in-memory CLI session tokens that never get a + token row. Their owner is written to LiteLLM_SpendLogs metadata at request + time under the same hashed api_key, so it is the only surviving source. Only + sha256 digests are looked up, matching the reverse-hash recovery gate, since + every current api_key value in spend logs is a token hash. The [start, end) + bound keeps the lookup on the startTime index instead of scanning the table. + """ + sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) + if not sha_missing: + return _EMPTY_KEY_METADATA + start, end = window + rows: Final = await _db_or_empty( + lambda: prisma_client.db.query_raw(_SPEND_LOG_ALIAS_SQL, sorted(sha_missing), start, end), + "Failed spend-log alias recovery for %d missing keys: %s", + len(sha_missing), + ) + if rows is None: + return _EMPTY_KEY_METADATA + return MappingProxyType( + { + row.digest: KeyMetadataDict(key_alias=row.key_alias, team_id=row.team_id, user_id=row.user_id) + for row in _TOKEN_DIGEST_ROWS.validate_python(rows) + if row.digest in sha_missing and (row.key_alias or row.user_id or row.team_id) + } + ) + + def _row_with_recovered_fields( row: Mapping[str, object], recovered: Mapping[str, KeyMetadataDict], diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 37a54c4901a..4d3dc320d8f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -2105,3 +2105,56 @@ async def test_get_daily_activity_aggregated_with_entity_breakdown(): # Rollups with the entity bit set must still land in their usual buckets assert daily.breakdown.models["gpt-4o"].metrics.spend == 18.0 assert daily.breakdown.api_keys["key-1"].metrics.spend == 12.0 + + +@pytest.mark.asyncio +async def test_get_api_key_metadata_resolves_session_key_via_spend_log_window(): + """ + A CLI session token has no verification-token row, so the active/deleted lookups + and reverse-hash all miss. Given a spend-log window, its alias and owner are + recovered from the spend-log metadata and its email is filled from the user table. + """ + from litellm.proxy.utils import hash_token + + session_digest = hash_token("cli-session-user-42") + mock_prisma = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_usertable.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="user-42", user_email="user42@example.com")] + ) + + async def query_raw(sql, *params): + if "LiteLLM_SpendLogs" in sql: + return [{"digest": session_digest, "key_alias": "cli-session-user-42", "team_id": None, "user_id": "user-42"}] + return [] + + mock_prisma.db.query_raw = AsyncMock(side_effect=query_raw) + + result = await get_api_key_metadata( + prisma_client=mock_prisma, + api_keys={session_digest}, + spend_logs_window=(datetime(2026, 9, 7), datetime(2026, 9, 10)), + ) + + assert result[session_digest]["key_alias"] == "cli-session-user-42" + assert result[session_digest]["user_id"] == "user-42" + assert result[session_digest]["user_email"] == "user42@example.com" + spend_log_calls = [call.args for call in mock_prisma.db.query_raw.call_args_list if "LiteLLM_SpendLogs" in call.args[0]] + ((_, digests, start, end),) = spend_log_calls + assert digests == [session_digest] + assert (start, end) == (datetime(2026, 9, 7), datetime(2026, 9, 10)) + + +def test_spend_logs_window_pads_min_minus_one_day_and_max_plus_two_days(): + from litellm.proxy.management_endpoints.common_daily_activity import _spend_logs_window + + window = _spend_logs_window({"2026-09-08", "2026-09-05", "not-a-date"}) + + assert window == (datetime(2026, 9, 4), datetime(2026, 9, 10)) + + +def test_spend_logs_window_is_none_when_no_date_parses(): + from litellm.proxy.management_endpoints.common_daily_activity import _spend_logs_window + + assert _spend_logs_window({"garbage", ""}) is None diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 7a80319239d..9e42f017106 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -1,4 +1,5 @@ from collections.abc import Sequence +from datetime import datetime from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -8,14 +9,24 @@ from prisma.errors import PrismaError from litellm.proxy.spend_tracking.key_metadata_recovery import ( fill_missing_api_key_aliases, recover_double_hashed_key_metadata, + recover_key_metadata_from_spend_logs, ) from litellm.proxy.utils import hash_token -def _digest_row(digest: str, key_alias: str, team_id: str | None, user_id: str | None) -> dict[str, str | None]: +def _digest_row(digest: str, key_alias: str | None, team_id: str | None, user_id: str | None) -> dict[str, str | None]: return {"digest": digest, "key_alias": key_alias, "team_id": team_id, "user_id": user_id} +def _query_raw_spend_logs(rows: Sequence[dict[str, str | None]]) -> AsyncMock: + async def query_raw(sql: str, *params: object) -> list[dict[str, str | None]]: + if '"LiteLLM_SpendLogs"' in sql: + return list(rows) + raise AssertionError(f"unexpected query: {sql}") + + return AsyncMock(side_effect=query_raw) + + def _query_raw_by_table( active_rows: Sequence[dict[str, str | None]], deleted_rows: Sequence[dict[str, str | None]], @@ -218,3 +229,85 @@ async def test_fill_missing_api_key_aliases_skips_named_keys_that_have_no_email( assert filled == rows mock_prisma.db.query_raw.assert_not_called() + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_resolves_session_token_from_metadata(): + """ + CLI session tokens never get a verification-token row, so both the exact join and + the reverse-hash lookup miss them. Their owner survives only in the spend-log + metadata written at request time, keyed by the same hashed api_key. + """ + session_digest = hash_token("cli-session-repro-user-6852") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs( + [_digest_row(session_digest, "cli-session-repro-user-6852", None, "repro-user-6852")] + ) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, {session_digest}, window) + + assert result[session_digest]["key_alias"] == "cli-session-repro-user-6852" + assert result[session_digest]["user_id"] == "repro-user-6852" + ((_, digests, start, end),) = [call.args for call in mock_prisma.db.query_raw.call_args_list] + assert digests == [session_digest] + assert (start, end) == window + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_skips_query_when_no_missing_keys(): + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, set(), window) + + assert result == {} + mock_prisma.db.query_raw.assert_not_called() + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_returns_empty_on_prisma_error(): + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock(side_effect=PrismaError("db down")) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, {hash_token("cli-session-x")}, window) + + assert result == {} + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_ignores_foreign_and_all_null_rows(): + wanted = hash_token("cli-session-wanted") + all_null = hash_token("cli-session-null") + foreign = hash_token("cli-session-foreign") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs( + [ + _digest_row(wanted, "kept-alias", None, "owner-1"), + _digest_row(all_null, None, None, None), + _digest_row(foreign, "foreign-alias", None, "owner-2"), + ] + ) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, {wanted, all_null}, window) + + assert set(result) == {wanted} + assert result[wanted]["key_alias"] == "kept-alias" + assert result[wanted]["user_id"] == "owner-1" + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_skips_non_sha256_keys(): + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + + result = await recover_key_metadata_from_spend_logs( + mock_prisma, {"cli-session-raw-1798", "key-hash-short"}, window + ) + + assert result == {} + mock_prisma.db.query_raw.assert_not_called() From c6f5763443504501e88113e783d2e1ec88b9264d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:04:52 -0700 Subject: [PATCH 105/411] feat(guardrails): run legacy post-call hooks as streaming pipeline steps A post_call pipeline step whose guardrail only implements the older async_post_call_success_hook used to skip the stream entirely: PR #38721 fails that shape open with a warning. The streaming step now assembles the buffered stream into the response the hook expects, runs the hook, ends the stream with the hook's exception when it raises, and delivers the hook's rewrite through the same event write-back the unified guardrails use on chat, Responses, and Messages streams (Messages gets the Anthropic shape). A stream a pipeline manages no longer runs the same hook again after the stream ends. A guardrail with neither the unified interface nor a post-call hook keeps the fail-open, as does a rewrite the buffer cannot be patched with. --- .../chat/guardrail_translation/handler.py | 5 + .../guardrail_translation/base_translation.py | 7 + litellm/proxy/common_request_processing.py | 15 +- .../proxy/policy_engine/pipeline_executor.py | 113 ++++++++- litellm/proxy/utils.py | 30 +-- .../test_anthropic_guardrail_handler.py | 26 +++ .../policy_engine/test_pipeline_executor.py | 150 +++++++++++- .../test_proxy_logging_hook_detection.py | 26 +++ .../proxy_logging/test_guardrail_pipeline.py | 220 +++++++++++++----- 9 files changed, 505 insertions(+), 87 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index e486be12fe2..f61cfe58e80 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -176,6 +176,11 @@ class AnthropicMessagesHandler(BaseTranslation): super().__init__() self.adapter = LiteLLMAnthropicMessagesAdapter() + def post_call_hook_response(self, response: object) -> object: + if not isinstance(response, ModelResponse): + return response + return self.adapter.translate_openai_response_to_anthropic(response) + @staticmethod def _build_streaming_usage_response( responses_so_far: Sequence[object], diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index afd8e0f67f7..bef472b2882 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -60,6 +60,13 @@ class BaseTranslation(ABC): text rewrites on every other translation, are undeliverable: the pipeline executor discards them and releases the original chunks.""" + def post_call_hook_response(self, response: object) -> object: + """The ``response`` this endpoint's non-streaming post-call hooks receive, derived from + the object the translation stores under ``request_data["response"]`` while scanning an + ended stream. Chat and Responses scan that shape already; a translation that scans a + different one (Messages scans an OpenAI-shaped ModelResponse) overrides this.""" + return response + @staticmethod def transform_user_api_key_dict_to_metadata( user_api_key_dict: Any | None, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9720e4b1cf8..7e85b66682c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3328,9 +3328,10 @@ class ProxyBaseLLMRequestProcessing: has completed. Guardrails routed through unified_guardrail are skipped, since they already ran - via its streaming iterator. Guardrails that override - async_post_call_success_hook directly run here, including those that implement - apply_guardrail but keep their native lifecycle hooks. + via its streaming iterator, and so are guardrails a post_call policy pipeline + manages, since the pipeline ran them against the buffered stream. Guardrails + that override async_post_call_success_hook directly run here, including those + that implement apply_guardrail but keep their native lifecycle hooks. This is audit-only — content has already been delivered to the client. @@ -3340,12 +3341,18 @@ class ProxyBaseLLMRequestProcessing: _response = assembled_response try: from litellm.proxy.proxy_server import llm_router as _global_llm_router - from litellm.proxy.utils import _check_and_merge_model_level_guardrails + from litellm.proxy.utils import ( + _check_and_merge_model_level_guardrails, + pipeline_managed_guardrail_names, + ) guardrail_data = _check_and_merge_model_level_guardrails(data=captured_data, llm_router=_global_llm_router) + pipeline_managed: Final = pipeline_managed_guardrail_names(captured_data, "post_call") for cb in litellm.callbacks: if not isinstance(cb, CustomGuardrail): continue + if cb.guardrail_name in pipeline_managed: + continue if not cb.should_run_guardrail( data=guardrail_data, event_type=GuardrailEventHooks.post_call, diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 9bc10949e9f..25c54d72f25 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -121,6 +121,80 @@ class _StreamRewriteObserver(CustomGuardrail): return outputs +class _ScannedTextRecorder(CustomGuardrail): + def __init__(self, guardrail_name: str) -> None: + super().__init__(guardrail_name=guardrail_name) + self.texts: tuple[str, ...] | None = None + + @_logged_by_inner_guardrail + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, # mutable-ok: matches CustomGuardrail.apply_guardrail + input_type: Literal["request", "response"], + logging_obj: "LiteLLMLoggingObj | None" = None, + ) -> GenericGuardrailAPIInputs: + self.texts = _text_snapshot(inputs.get("texts")) + return inputs + + +class _LegacyHookStreamAdapter(CustomGuardrail): + """Runs a guardrail that only implements the legacy post-call hook (no unified + ``apply_guardrail``, or ``use_native_lifecycle_hooks``) as a streaming pipeline step. The + endpoint translation hands it the texts it scanned plus the assembled response under + ``request_data["response"]``; the hook gets that response in the shape its route gives + non-streaming hooks, an exception it raises ends the stream through the executor's + fail/error classification, and a replacement response is re-scanned by the same translation + so its texts reach the client through the translation's ended-stream write-back. A + replacement whose scanned texts do not line up with the originals is undeliverable, so the + executor releases the original chunks.""" + + def __init__( + self, + inner: CustomGuardrail, + endpoint_translation: "BaseTranslation", + user_api_key_dict: "UserAPIKeyAuth", + ) -> None: + super().__init__(guardrail_name=inner.guardrail_name) + self.inner: Final = inner + self.endpoint_translation: Final = endpoint_translation + self.user_api_key_dict: Final = user_api_key_dict + + def structured_messages_cover_full_request(self) -> bool: + return self.inner.structured_messages_cover_full_request() + + @_logged_by_inner_guardrail + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, # mutable-ok: matches CustomGuardrail.apply_guardrail + input_type: Literal["request", "response"], + logging_obj: "LiteLLMLoggingObj | None" = None, + ) -> GenericGuardrailAPIInputs: + replacement: Final = await self.inner.async_post_call_success_hook( + data=request_data, + user_api_key_dict=self.user_api_key_dict, + response=self.endpoint_translation.post_call_hook_response(request_data.get("response")), + ) + if replacement is None: + return inputs + scanned: Final = _text_snapshot(inputs.get("texts")) + rewritten: Final = await self._scanned_texts(replacement, logging_obj) + if scanned is None or rewritten is None or len(rewritten) != len(scanned): + raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") + return {**inputs, "texts": list(rewritten)} + + async def _scanned_texts(self, response: object, logging_obj: "LiteLLMLoggingObj | None") -> tuple[str, ...] | None: + recorder: Final = _ScannedTextRecorder(self.guardrail_name or "unknown") + await self.endpoint_translation.process_output_response( + response=response, + guardrail_to_apply=recorder, + litellm_logging_obj=logging_obj, + user_api_key_dict=self.user_api_key_dict, + ) + return recorder.texts + + def _prepare_hook_input( step: PipelineStep, callback: CustomGuardrail, @@ -286,16 +360,23 @@ class PipelineExecutor: endpoint_translation: "BaseTranslation", streaming_chunks: list[object], # mutable-ok: shared buffered-stream chunks the translation rewrites in place hook_input: dict[str, object], # mutable-ok: same request-payload shape as data - user_api_key_dict: "UserAPIKeyAuth | None", + user_api_key_dict: "UserAPIKeyAuth", litellm_logging_obj: "LiteLLMLoggingObj | None", ) -> None: """Run one streaming post_call step through the endpoint translation, delivering - text rewrites on translations that support ended-stream write-back. A rewrite that - cannot reach the client yet (a tool-call rewrite, a text rewrite on a translation - without write-back, or one the translation refused with - ``UndeliverableStreamRewrite``) is discarded: the buffered chunks go back to the - originals and the step passes, so the client gets the stream the merge base sent.""" - observer: Final = _StreamRewriteObserver(callback) + text rewrites on translations that support ended-stream write-back. A guardrail + without the unified interface runs its legacy post-call hook against the assembled + response through ``_LegacyHookStreamAdapter``. A rewrite that cannot reach the client + yet (a tool-call rewrite, a text rewrite on a translation without write-back, or one + the translation or adapter refused with ``UndeliverableStreamRewrite``) is discarded: + the buffered chunks go back to the originals and the step passes, so the client gets + the stream the merge base sent.""" + scanner: Final = ( + callback + if PipelineExecutor.supports_unified_execution(callback) + else _LegacyHookStreamAdapter(callback, endpoint_translation, user_api_key_dict) + ) + observer: Final = _StreamRewriteObserver(scanner) deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_text_rewrites originals: Final = copy.deepcopy(streaming_chunks) try: @@ -379,11 +460,11 @@ class PipelineExecutor: if isinstance(response, dict): callback.mark_pre_call_hook_ran(response) elif mode == "post_call" and streaming_chunks is not None: - if not use_unified or endpoint_translation is None: + if endpoint_translation is None: return ( "error", None, - f"Guardrail '{step.guardrail}' does not support streaming pipeline execution", + f"Guardrail '{step.guardrail}' cannot run on a stream without an endpoint translation", None, ) await PipelineExecutor._run_streaming_step( @@ -433,10 +514,20 @@ class PipelineExecutor: @staticmethod def supports_unified_execution(callback: CustomGuardrail) -> bool: - """Whether this guardrail runs through the unified apply_guardrail path, - the interface streaming pipeline execution requires.""" + """Whether this guardrail runs through the unified apply_guardrail path.""" return "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks + @staticmethod + def supports_streaming_execution(callback: CustomGuardrail) -> bool: + """Whether a streaming pipeline step can run this guardrail against the buffered + stream: through the unified path, or through its own post-call hook on the + assembled response. A guardrail with neither (one that only rewrites the stream + through its iterator hook) has to keep running on its own.""" + return ( + PipelineExecutor.supports_unified_execution(callback) + or type(callback).async_post_call_success_hook is not CustomLogger.async_post_call_success_hook + ) + @staticmethod def find_guardrail_callback(guardrail_name: str) -> CustomGuardrail | None: """Look up an initialized guardrail callback by name from litellm.callbacks.""" diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8b36061c6be..bbec1b842c4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -451,7 +451,7 @@ def _pipeline_step_guardrail_names(pipelines: Sequence[tuple[str, "GuardrailPipe return frozenset(step.guardrail for _policy_name, pipeline in pipelines for step in pipeline.steps) -def _pipeline_managed_guardrail_names( +def pipeline_managed_guardrail_names( data: Mapping[str, object], mode: Literal["pre_call", "post_call"] ) -> frozenset[str]: return _pipeline_step_guardrail_names( @@ -514,9 +514,9 @@ def _merge_pipeline_metadata_writes( _merge_pipeline_metadata_bucket(data, bucket_key, modified_data.get(bucket_key)) -def _pipeline_step_supports_unified_streaming(guardrail_name: str) -> bool: +def _pipeline_step_supports_streaming(guardrail_name: str) -> bool: callback: Final = PipelineExecutor.find_guardrail_callback(guardrail_name) - return callback is not None and PipelineExecutor.supports_unified_execution(callback) + return callback is not None and PipelineExecutor.supports_streaming_execution(callback) def _post_call_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "GuardrailPipeline"], ...]: @@ -541,14 +541,15 @@ def _warn_background_skips_post_call_pipelines(data: Mapping[str, object]) -> No def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: unsupported: Final = tuple( dict.fromkeys( - step.guardrail for step in pipeline.steps if not _pipeline_step_supports_unified_streaming(step.guardrail) + step.guardrail for step in pipeline.steps if not _pipeline_step_supports_streaming(step.guardrail) ) ) if not unsupported: return True verbose_proxy_logger.warning( - "Policy '%s' has post_call pipeline guardrails without the unified apply_guardrail interface, " - "which streaming pipelines need; the stream skips the pipeline and its guardrails run on their own: %s", + "Policy '%s' has post_call pipeline guardrails with neither the unified apply_guardrail interface nor a " + "post-call hook, one of which streaming pipelines need; the stream skips the pipeline and its guardrails " + "run on their own: %s", policy_name, ", ".join(unsupported), ) @@ -562,11 +563,12 @@ def _streamable_post_call_pipelines( The post_call pipelines a streaming response can be gated through. Streaming pipelines scan the buffered stream through the endpoint guardrail - translation of the request route, so every step's guardrail needs the - unified apply_guardrail interface and the route needs a translation. A - pipeline that cannot be run that way yet is left out and its guardrails - run on the stream on their own, the way they did before pipelines ran on - streams at all, with a warning naming the pipeline. + translation of the request route, so every step's guardrail needs either the + unified apply_guardrail interface or a post-call hook to run against the + assembled response, and the route needs a translation. A pipeline that + cannot be run that way yet is left out and its guardrails run on the stream + on their own, the way they did before pipelines ran on streams at all, with + a warning naming the pipeline. """ post_call_pipelines: Final = _post_call_pipelines(request_data) if not post_call_pipelines: @@ -1968,7 +1970,7 @@ class ProxyLogging: ) # Get pipeline-managed guardrails to skip in normal loop - pipeline_managed: Final = _pipeline_managed_guardrail_names(data, "pre_call") + pipeline_managed: Final = pipeline_managed_guardrail_names(data, "pre_call") caps: Final = ProxyLogging._callback_capabilities() # Skip the per-request callback walk entirely when nothing in @@ -2956,7 +2958,7 @@ class ProxyLogging: if pipeline_response is not None: response = pipeline_response # rebind-ok: adopt the pipeline's replacement response, same contract as the callback loops below - pipeline_managed: Final = _pipeline_managed_guardrail_names(data, "post_call") + pipeline_managed: Final = pipeline_managed_guardrail_names(data, "post_call") guardrail_callbacks, other_callbacks = _partition_post_call_callbacks() try: # Merge model-level guardrails before checking which guardrails to run @@ -3272,7 +3274,7 @@ class ProxyLogging: _cached_guardrail_data: dict | None = None _guardrail_data_computed = False pipeline_managed: Final = ( - _pipeline_managed_guardrail_names(data, "post_call") if caps.has_guardrail else frozenset() + pipeline_managed_guardrail_names(data, "post_call") if caps.has_guardrail else frozenset() ) for callback in litellm.callbacks: diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index bf40f781fa3..a10a2aae0ee 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -2156,3 +2156,29 @@ class TestAnthropicMessagesHandlerStreamingScanKey: assert open_key == StreamingScanKey(texts=("hi",)) assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] assert ended_key != open_key + + +class TestAnthropicMessagesHandlerPostCallHookResponse: + def test_openai_shaped_stream_assembly_reaches_the_hook_as_a_messages_response(self): + from litellm.types.utils import Choices, Message, ModelResponse, Usage + + assembled = ModelResponse( + id="msg_1", + model="claude", + choices=[Choices(message=Message(role="assistant", content="hello world"), finish_reason="stop")], + usage=Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3), + ) + + hook_response = AnthropicMessagesHandler().post_call_hook_response(assembled) + + assert hook_response["type"] == "message" + assert hook_response["role"] == "assistant" + assert hook_response["content"] == [{"type": "text", "text": "hello world"}] + assert hook_response["stop_reason"] == "end_turn" + assert hook_response["usage"]["input_tokens"] == 1 + assert hook_response["usage"]["output_tokens"] == 2 + + def test_anything_else_reaches_the_hook_untouched(self): + native = {"type": "message", "role": "assistant", "content": [{"type": "text", "text": "hi"}]} + + assert AnthropicMessagesHandler().post_call_hook_response(native) is native diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 54ef1f79f4d..db403a17c28 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -534,7 +534,6 @@ async def test_guardrail_not_found_uses_on_fail(monkeypatch): ], ) - monkeypatch.setattr(litellm, "callbacks", []) result = await PipelineExecutor.execute_steps( steps=pipeline.steps, @@ -1153,3 +1152,152 @@ async def test_streaming_step_restores_chunks_when_translation_refuses_the_rewri _assert_passed_with_discard_warning(result, caplog) assert chunks == [_chunk()] + + +class _LegacyHookGuardrail(CustomGuardrail): + """A guardrail with only the legacy post-call hook: it never defines apply_guardrail.""" + + def __init__(self, replacement=None, raises=None): + super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True) + self.replacement = replacement + self.raises = raises + self.calls = [] + + async def async_post_call_success_hook(self, data, user_api_key_dict, response): + self.calls.append({"data": data, "user_api_key_dict": user_api_key_dict, "response": response}) + if self.raises is not None: + raise self.raises + return self.replacement + + +class _NativeHooksGuardrail(_LegacyHookGuardrail): + use_native_lifecycle_hooks = True + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + raise AssertionError("a guardrail that keeps its native hooks never runs apply_guardrail") + + +class _LegacyScanningTranslation: + """Stores the assembled response under request_data["response"] before scanning, like the + chat, Responses, and Messages handlers, hands hooks a route-native shape, and re-extracts one + text per entry of a replacement's "texts".""" + + delivers_ended_stream_text_rewrites = True + + def post_call_hook_response(self, response): + return {"native": True, "text": response["text"]} + + async def process_output_streaming_response( + self, + responses_so_far, + guardrail_to_apply, + litellm_logging_obj=None, + user_api_key_dict=None, + request_data=None, + deliver_ended_stream_rewrites=False, + ): + request_data.setdefault("response", {"text": responses_so_far[0]["text"]}) + outputs = await guardrail_to_apply.apply_guardrail( + inputs={"texts": [responses_so_far[0]["text"]]}, + request_data=request_data, + input_type="response", + logging_obj=litellm_logging_obj, + ) + responses_so_far[0]["text"] = outputs["texts"][0] + return responses_so_far + + async def process_output_response( + self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None + ): + await guardrail_to_apply.apply_guardrail( + inputs={"texts": list(response["texts"])}, + request_data={"response": response}, + input_type="response", + logging_obj=litellm_logging_obj, + ) + return response + + +async def _run_legacy_streaming_step(monkeypatch, guardrail, chunks, on_fail="block", on_error="next"): + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + return await PipelineExecutor.execute_steps( + steps=[PipelineStep(guardrail="masker", on_pass="allow", on_fail=on_fail, on_error=on_error)], + mode="post_call", + data={"model": "m"}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="p", + streaming_chunks=chunks, + endpoint_translation=_LegacyScanningTranslation(), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("guardrail_class", [_LegacyHookGuardrail, _NativeHooksGuardrail]) +async def test_streaming_step_runs_legacy_hook_and_delivers_its_rewrite(monkeypatch, caplog, guardrail_class): + guardrail = guardrail_class(replacement={"texts": ["[REWRITTEN] hello world"]}) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert chunks[0]["text"] == "[REWRITTEN] hello world" + assert [call["response"] for call in guardrail.calls] == [{"native": True, "text": "hello world"}] + assert guardrail.calls[0]["data"]["model"] == "m" + assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_passes_untouched_when_legacy_hook_returns_none(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement=None) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + assert result.terminal_action == "allow" + assert len(guardrail.calls) == 1 + assert chunks == [_chunk()] + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") +@pytest.mark.asyncio +async def test_streaming_step_blocks_with_the_legacy_hook_exception(monkeypatch): + exc = HTTPException(status_code=400, detail={"error": "output blocked"}) + chunks = [_chunk()] + + result = await _run_legacy_streaming_step(monkeypatch, _LegacyHookGuardrail(raises=exc), chunks) + + assert result.terminal_action == "block" + assert [step.outcome for step in result.step_results] == ["fail"] + assert result.original_exception is exc + assert chunks == [_chunk()] + + +@pytest.mark.asyncio +async def test_streaming_step_takes_on_error_when_legacy_hook_crashes(monkeypatch): + chunks = [_chunk()] + + result = await _run_legacy_streaming_step( + monkeypatch, _LegacyHookGuardrail(raises=ValueError("boom")), chunks, on_error="block" + ) + + assert result.terminal_action == "block" + assert [step.outcome for step in result.step_results] == ["error"] + assert result.step_results[0].error_detail == "boom" + + +@pytest.mark.asyncio +async def test_streaming_step_discards_legacy_rewrite_whose_texts_do_not_line_up(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement={"texts": ["split", "in two"]}) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index 9f1321aec2c..f4165f98ba5 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -671,6 +671,32 @@ async def test_deferred_stream_guardrails_run_native_hook_when_opted_out(monkeyp assert routed.native_hooks_ran == [] +@pytest.mark.asyncio +async def test_deferred_stream_guardrails_skip_pipeline_managed_native_hook(monkeypatch): + """A post_call pipeline step already ran the opted-out guardrail's own hook against + the buffered stream, so the deferred audit must not run it a second time.""" + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + from litellm.types.utils import Choices, Message, ModelResponse + + pipeline_managed = _KeepsNativeHooks(event_hook=GuardrailEventHooks.post_call, default_on=True) + monkeypatch.setattr(litellm, "callbacks", [pipeline_managed]) + pipeline = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="keeps_native", on_fail="block")]) + + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={ + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"_guardrail_pipelines": [("response-governance", pipeline)]}, + }, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234"), + captured_logging_obj=_streaming_logging_obj(), + assembled_response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="hello"))]), + cache_hit=False, + ) + + assert pipeline_managed.native_hooks_ran == [] + + @pytest.mark.asyncio async def test_realtime_guardrails_skip_opted_out_guardrail(monkeypatch): """The realtime path calls apply_guardrail directly, so the opt-out has to be diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 73dd6746b29..d01f881c446 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -11,6 +11,7 @@ from __future__ import annotations import asyncio import json +from copy import deepcopy import logging from typing import Any, Callable, Dict, List from unittest.mock import AsyncMock, MagicMock, patch @@ -1497,29 +1498,78 @@ async def _async_chunk_iter(chunks: List[Any]): yield chunk -def test_streamable_post_call_pipelines_keeps_supported_and_drops_unsupported( +def _legacy_hook_stream_guardrail( + seen: Dict[str, Any], + rewrite: Callable[[Any], Any] | None = None, + raises: Exception | None = None, + native_lifecycle: bool = False, +) -> CustomGuardrail: + class LegacyHookGuardrail(CustomGuardrail): + use_native_lifecycle_hooks = native_lifecycle + + async def async_post_call_success_hook(self, data, user_api_key_dict, response): + seen["count"] = seen.get("count", 0) + 1 + seen["data"] = data + seen["user_api_key_dict"] = user_api_key_dict + seen["response"] = deepcopy(response) + if raises is not None: + raise raises + return None if rewrite is None else rewrite(response) + + if native_lifecycle: + + class NativeLifecycleGuardrail(LegacyHookGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + raise AssertionError("a guardrail that keeps its native hooks never runs apply_guardrail") + + return NativeLifecycleGuardrail( + guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False + ) + return LegacyHookGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False) + + +def _iterator_hook_only_guardrail(name: str, seen: Dict[str, Any]) -> CustomGuardrail: + class IteratorHookGuardrail(CustomGuardrail): + async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): + seen["count"] = seen.get("count", 0) + 1 + async for item in response: + item.choices[0].delta.content = f"[governed] {item.choices[0].delta.content}" + yield item + + return IteratorHookGuardrail(guardrail_name=name, event_hook=GuardrailEventHooks.post_call, default_on=True) + + +def _rewritten_model_response(response: Any) -> litellm.ModelResponse: + payload = response.model_dump() + payload["choices"][0]["message"]["content"] = "[REWRITTEN] " + payload["choices"][0]["message"]["content"] + return litellm.ModelResponse(**payload) + + +def test_streamable_post_call_pipelines_keeps_hook_guardrails_and_drops_iterator_only( make_user_api_key_auth, monkeypatch, caplog ): - class NativeOnlyGuardrail(CustomGuardrail): - pass - supported = _unified_stream_guardrail({}) - native_only = NativeOnlyGuardrail(guardrail_name="gr-native", event_hook=GuardrailEventHooks.post_call) - monkeypatch.setattr(litellm, "callbacks", [supported, native_only]) - governed = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")]) + legacy = _legacy_hook_stream_guardrail({}) + legacy.guardrail_name = "gr-legacy" + iterator_only = _iterator_hook_only_guardrail("gr-iterator", {}) + monkeypatch.setattr(litellm, "callbacks", [supported, legacy, iterator_only]) + governed = GuardrailPipeline( + mode="post_call", + steps=[PipelineStep(guardrail="gr-post", on_fail="next"), PipelineStep(guardrail="gr-legacy", on_fail="block")], + ) ungoverned = GuardrailPipeline( mode="post_call", - steps=[PipelineStep(guardrail="gr-post", on_fail="next"), PipelineStep(guardrail="gr-native", on_fail="block")], + steps=[PipelineStep(guardrail="gr-post", on_fail="next"), PipelineStep(guardrail="gr-iterator", on_fail="block")], ) - pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="gr-native", on_fail="block")]) + pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="gr-iterator", on_fail="block")]) data = {"metadata": {"_guardrail_pipelines": [("governed", governed), ("ungoverned", ungoverned), ("req", pre_call)]}} with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): streamable = _streamable_post_call_pipelines(data, make_user_api_key_auth(request_route="/v1/chat/completions")) assert streamable == (("governed", governed),) - assert any("'ungoverned'" in message and "gr-native" in message for message in _warnings(caplog)) - assert not any("'governed'" in message for message in _warnings(caplog)) + assert any("'ungoverned'" in message and "gr-iterator" in message for message in _warnings(caplog)) + assert not any("'governed'" in message or "gr-legacy" in message for message in _warnings(caplog)) def test_streamable_post_call_pipelines_is_empty_on_route_without_translation( @@ -1569,55 +1619,123 @@ async def test_pre_call_hook_allows_streaming_when_pipeline_guardrail_supports_u @pytest.mark.asyncio @pytest.mark.parametrize("native_lifecycle", [False, True]) -async def test_streaming_iterator_hook_releases_stream_when_pipeline_guardrail_lacks_unified_support( +async def test_streaming_iterator_hook_runs_legacy_hook_and_delivers_its_rewrite( proxy_logging, make_user_api_key_auth, monkeypatch, native_lifecycle, caplog ): seen: Dict[str, Any] = {} - if native_lifecycle: - - class NativeOnlyGuardrail(CustomGuardrail): - use_native_lifecycle_hooks = True - - async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): - seen["count"] = seen.get("count", 0) + 1 - return inputs - - else: - - class NativeOnlyGuardrail(CustomGuardrail): - async def async_post_call_success_hook(self, data, user_api_key_dict, response): - seen["count"] = seen.get("count", 0) + 1 - return response - - monkeypatch.setattr( - litellm, - "callbacks", - [NativeOnlyGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)], - ) + guardrail = _legacy_hook_stream_guardrail(seen, rewrite=_rewritten_model_response, native_lifecycle=native_lifecycle) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _post_call_pipeline_data(stream=True) chunks = _stream_chunks() - delivered: List[Any] = [] + auth = make_user_api_key_auth(request_route="/v1/chat/completions") with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): out = await proxy_logging.pre_call_hook( - user_api_key_dict=make_user_api_key_auth(), - data=data, - call_type="completion", - guardrails_only=True, + user_api_key_dict=auth, data=data, call_type="completion", guardrails_only=True ) + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=auth, response=_async_chunk_iter(chunks), request_data=data + ) + ] + + assert out is not None and out.get("stream") is True + assert seen["count"] == 1 + assert isinstance(seen["response"], litellm.ModelResponse) + assert seen["response"].choices[0].message.content == "hello world" + assert seen["data"]["messages"] == data["messages"] + assert seen["user_api_key_dict"] is auth + assert [id(item) for item in delivered] == [id(chunk) for chunk in chunks] + assert delivered[0].choices[0].delta.content == "[REWRITTEN] hello world" + assert delivered[1].choices[0].delta.content in (None, "") + assert delivered[1].choices[0].finish_reason == "stop" + assert data["metadata"]["applied_guardrails"] == ["gr-post"] + assert _warnings(caplog) == [] + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_releases_stream_untouched_when_legacy_hook_returns_none( + proxy_logging, make_user_api_key_auth, monkeypatch +): + seen: Dict[str, Any] = {} + monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail(seen)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + chunks = _stream_chunks() + + delivered = [ + item async for item in proxy_logging.async_post_call_streaming_iterator_hook( user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"), response=_async_chunk_iter(chunks), request_data=data, + ) + ] + + assert seen["count"] == 1 + assert [id(item) for item in delivered] == [id(chunk) for chunk in chunks] + assert [item.choices[0].delta.content for item in delivered] == ["hello ", "world"] + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_ends_stream_with_legacy_hook_exception( + proxy_logging, make_user_api_key_auth, monkeypatch +): + seen: Dict[str, Any] = {} + blocked = HTTPException(status_code=400, detail={"error": "output blocked"}) + monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail(seen, raises=blocked)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + delivered: List[Any] = [] + + async def _drain() -> None: + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"), + response=_async_chunk_iter(_stream_chunks()), + request_data=data, ): delivered.append(item) - assert out is not None - assert out.get("stream") is True - assert [item is chunk for item, chunk in zip(delivered, chunks)] == [True, True] - assert len(delivered) == 2 - assert seen.get("count") is None - assert any("'response-governance'" in message and "gr-post" in message for message in _warnings(caplog)) + with pytest.raises(HTTPException) as info: + await _drain() + + assert seen["count"] == 1 + assert delivered == [] + assert info.value is blocked + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_delivers_legacy_hook_rewrite_on_anthropic_sse( + proxy_logging, make_user_api_key_auth, monkeypatch +): + seen: Dict[str, Any] = {} + + def rewrite(response: Any) -> Dict[str, Any]: + return {**response, "content": [{"type": "text", "text": "[REWRITTEN] " + response["content"][0]["text"]}]} + + monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail(seen, rewrite=rewrite)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"), + response=_async_chunk_iter(_anthropic_sse_chunks()), + request_data=data, + ) + ] + + assert seen["count"] == 1 + assert seen["response"]["content"][0]["text"] == "hello world" + assert seen["response"]["role"] == "assistant" + raw = b"".join(delivered).decode() + assert "[REWRITTEN] hello world" in raw + assert raw.count("event: content_block_delta") == 1 + for expected_event in ("message_start", "content_block_start", "content_block_stop", "message_delta", "message_stop"): + assert f"event: {expected_event}" in raw @pytest.mark.asyncio @@ -1625,19 +1743,7 @@ async def test_streaming_iterator_hook_runs_iterator_hook_guardrail_whose_pipeli proxy_logging, make_user_api_key_auth, monkeypatch, caplog ): seen: Dict[str, Any] = {} - - class IteratorHookGuardrail(CustomGuardrail): - async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): - seen["count"] = seen.get("count", 0) + 1 - async for item in response: - item.choices[0].delta.content = f"[governed] {item.choices[0].delta.content}" - yield item - - monkeypatch.setattr( - litellm, - "callbacks", - [IteratorHookGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True)], - ) + monkeypatch.setattr(litellm, "callbacks", [_iterator_hook_only_guardrail("gr-post", seen)]) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _post_call_pipeline_data(stream=True) From db59e99932f9438ac364c731cb1584092a76c94d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:21:34 -0700 Subject: [PATCH 106/411] fix(azure): let the caller's api-version override a full-URL api_base on relays --- .../llms/azure/passthrough/transformation.py | 9 +++++- .../test_azure_passthrough_transformation.py | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 85647ae02c9..dc71c53e126 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -65,6 +65,12 @@ def logged_responses_stream(all_chunks: Sequence[str], logging_obj: Logging) -> return terminal_event +def without_api_version(api_base: str) -> str: + url: Final = httpx.URL(api_base) + kept_params: Final = tuple((key, value) for key, value in url.params.multi_items() if key != "api-version") + return str(url.copy_with(params=httpx.QueryParams(kept_params))) + + class AzurePassthroughConfig(BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: return bool(request_data.get("stream")) @@ -89,8 +95,9 @@ class AzurePassthroughConfig(BasePassthroughConfig): native_endpoint: Final = strip_leading_model_segment(routed_endpoint, (model,)) caller_api_version: Final = request_query_params.get("api-version") if request_query_params else None + relay_base: Final = without_api_version(base_target_url) if caller_api_version else base_target_url complete_url: Final = BaseAzureLLM._get_base_azure_url( - api_base=base_target_url, + api_base=relay_base, litellm_params={**litellm_params, "api_version": caller_api_version or litellm_params.get("api_version")}, route=native_endpoint, ) diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 6e85b0cdca6..a3f5c415417 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -369,6 +369,38 @@ def test_azure_passthrough_url_fills_in_the_deployments_api_version_when_the_cal assert url.params["api-version"] == "2024-10-21" +FULL_URL_API_BASE = ( + "https://my-resource.openai.azure.com/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2024-10-21" +) + + +def _full_url_complete_url(request_query_params: dict) -> httpx.URL: + url, _ = AzurePassthroughConfig().get_complete_url( + api_base=FULL_URL_API_BASE, + api_key="key", + model="gpt-4.1-mini", + endpoint="chat/completions", + request_query_params=request_query_params, + litellm_params={}, + ) + return url + + +def test_azure_passthrough_url_prefers_the_callers_api_version_over_a_full_url_api_bases(): + url = _full_url_complete_url(request_query_params={"api-version": "2025-04-01-preview"}) + + assert str(url) == ( + "https://my-resource.openai.azure.com/openai/deployments/gpt-4.1-mini/chat/completions" + "?api-version=2025-04-01-preview" + ) + + +def test_azure_passthrough_url_keeps_a_full_url_api_bases_api_version_when_the_caller_sends_none(): + url = _full_url_complete_url(request_query_params={}) + + assert url.params["api-version"] == "2024-10-21" + + def test_azure_passthrough_url_strips_the_leading_router_model_segment(): url, _ = AzurePassthroughConfig().get_complete_url( api_base="https://my-resource.openai.azure.com", From d497434680186367ce42795ae1fca314503cd2aa Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 8 Sep 2026 19:28:04 +0000 Subject: [PATCH 107/411] fix(model_prices): update provider and web search pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 140 +++++++++++++++--- model_prices_and_context_window.json | 140 +++++++++++++++--- .../test_tool_call_cost_tracking.py | 65 +++++++- ...tex_ai_gemini_transcribe_transformation.py | 4 +- 4 files changed, 293 insertions(+), 56 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ef8da798922..a540df1c7d1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29176,7 +29176,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, + "supports_web_search": true }, "gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, @@ -29193,9 +29199,9 @@ "output_cost_per_token_priority": 1e-06, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29204,7 +29210,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-4o-mini-audio-preview": { "deprecation_date": "2026-05-07", @@ -29294,9 +29301,9 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29321,9 +29328,9 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29434,9 +29441,9 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.05, - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035 + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29461,9 +29468,9 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.05, - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035 + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -41625,6 +41632,28 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "rerank-v4.0-fast": { + "input_cost_per_query": 0.002, + "input_cost_per_token": 0.0, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/pricing" + }, + "rerank-v4.0-pro": { + "input_cost_per_query": 0.0025, + "input_cost_per_token": 0.0, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/pricing" + }, "nvidia_nim/nvidia/nv-rerankqa-mistral-4b-v3": { "input_cost_per_query": 0.0, "input_cost_per_token": 0.0, @@ -47811,7 +47840,7 @@ "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", - "max_input_tokens": 2000000, + "max_input_tokens": 128000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", @@ -47827,7 +47856,7 @@ "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", - "max_input_tokens": 2000000, + "max_input_tokens": 128000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", @@ -47842,14 +47871,17 @@ }, "vertex_ai/xai/grok-4.20-non-reasoning": { "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/developers/models", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -47858,14 +47890,17 @@ }, "vertex_ai/xai/grok-4.20-reasoning": { "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/developers/models", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -47873,6 +47908,42 @@ "supports_vision": true, "supports_web_search": true }, + "vertex_ai/xai/grok-4.3": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vertex_ai/xai/grok-4.6": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 524288, + "max_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas": { "input_cost_per_token": 2.2e-07, "litellm_provider": "vertex_ai-qwen_models", @@ -56824,8 +56895,8 @@ "rpm": 10 }, "vertex_ai/gemini-3.5-transcribe-preview": { - "input_cost_per_audio_token": 2.5e-06, - "input_cost_per_token": 2.5e-06, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token": 2e-06, "litellm_provider": "vertex_ai", "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, @@ -56860,6 +56931,27 @@ ], "supports_audio_input": true }, + "vertex_ai/gemini-3.5-live-translate-preview": { + "input_cost_per_audio_token": 3.5e-06, + "input_cost_per_token": 3.5e-06, + "litellm_provider": "vertex_ai", + "mode": "realtime", + "output_cost_per_audio_token": 2.1e-05, + "output_cost_per_token": 2.1e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, "perplexity/pplx-embed-context-v1-0.6b": { "input_cost_per_token": 8e-09, "litellm_provider": "perplexity", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ef8da798922..a540df1c7d1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29176,7 +29176,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, + "supports_web_search": true }, "gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, @@ -29193,9 +29199,9 @@ "output_cost_per_token_priority": 1e-06, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29204,7 +29210,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-4o-mini-audio-preview": { "deprecation_date": "2026-05-07", @@ -29294,9 +29301,9 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29321,9 +29328,9 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { - "search_context_size_high": 0.03, + "search_context_size_high": 0.025, "search_context_size_low": 0.025, - "search_context_size_medium": 0.0275 + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29434,9 +29441,9 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.05, - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035 + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -29461,9 +29468,9 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.05, - "search_context_size_low": 0.03, - "search_context_size_medium": 0.035 + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 }, "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -41625,6 +41632,28 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "rerank-v4.0-fast": { + "input_cost_per_query": 0.002, + "input_cost_per_token": 0.0, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/pricing" + }, + "rerank-v4.0-pro": { + "input_cost_per_query": 0.0025, + "input_cost_per_token": 0.0, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/pricing" + }, "nvidia_nim/nvidia/nv-rerankqa-mistral-4b-v3": { "input_cost_per_query": 0.0, "input_cost_per_token": 0.0, @@ -47811,7 +47840,7 @@ "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", - "max_input_tokens": 2000000, + "max_input_tokens": 128000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", @@ -47827,7 +47856,7 @@ "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", - "max_input_tokens": 2000000, + "max_input_tokens": 128000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", @@ -47842,14 +47871,17 @@ }, "vertex_ai/xai/grok-4.20-non-reasoning": { "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/developers/models", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -47858,14 +47890,17 @@ }, "vertex_ai/xai/grok-4.20-reasoning": { "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, "max_tokens": 2000000, "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/developers/models", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -47873,6 +47908,42 @@ "supports_vision": true, "supports_web_search": true }, + "vertex_ai/xai/grok-4.3": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 200000, + "max_tokens": 200000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vertex_ai/xai/grok-4.6": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 524288, + "max_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas": { "input_cost_per_token": 2.2e-07, "litellm_provider": "vertex_ai-qwen_models", @@ -56824,8 +56895,8 @@ "rpm": 10 }, "vertex_ai/gemini-3.5-transcribe-preview": { - "input_cost_per_audio_token": 2.5e-06, - "input_cost_per_token": 2.5e-06, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_token": 2e-06, "litellm_provider": "vertex_ai", "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, @@ -56860,6 +56931,27 @@ ], "supports_audio_input": true }, + "vertex_ai/gemini-3.5-live-translate-preview": { + "input_cost_per_audio_token": 3.5e-06, + "input_cost_per_token": 3.5e-06, + "litellm_provider": "vertex_ai", + "mode": "realtime", + "output_cost_per_audio_token": 2.1e-05, + "output_cost_per_token": 2.1e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, "perplexity/pplx-embed-context-v1-0.6b": { "input_cost_per_token": 8e-09, "litellm_provider": "perplexity", diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 6cc3dcceebc..38bf8c544c7 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,5 +1,6 @@ -import os +import json from collections.abc import Mapping, Sequence +from pathlib import Path import pytest @@ -11,8 +12,6 @@ from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebS from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams - - def test_web_search_cost_low(): web_search_options = WebSearchOptions(search_context_size="low") model_info = litellm.get_model_info("gpt-4o-search-preview") @@ -683,12 +682,13 @@ def test_web_search_provider_prefix_fallback_does_not_misprice_non_gemini_model( def _openai_responses_with_web_search_calls(model, num_calls): - from litellm.types.llms.openai import ResponsesAPIResponse from openai.types.responses.response_function_web_search import ( ActionSearch, ResponseFunctionWebSearch, ) + from litellm.types.llms.openai import ResponsesAPIResponse + output = [ ResponseFunctionWebSearch( id=f"ws_{i}", @@ -859,11 +859,64 @@ def test_dated_search_preview_entries_carry_search_pricing(local_model_cost_map) custom_llm_provider="openai", standard_built_in_tools_params=None, ) - assert cost == pytest.approx(0.035), ( - f"dated search-preview id must bill the $0.035 search fee, got ${cost}" + assert cost == pytest.approx(0.025), ( + f"dated search-preview id must bill the $0.025 search fee, got ${cost}" ) +@pytest.mark.parametrize( + "web_search_options", + [ + None, + WebSearchOptions(search_context_size="low"), + WebSearchOptions(search_context_size="medium"), + WebSearchOptions(search_context_size="high"), + ], +) +def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias( + web_search_options, local_model_cost_map +): + """Snapshot and alias must bill web search identically: OpenAI lists preview search at $25 per 1k calls.""" + alias_info = litellm.get_model_info("gpt-4o-mini") + snapshot_info = litellm.get_model_info("gpt-4o-mini-2024-07-18") + + assert snapshot_info["supports_web_search"] is True + assert alias_info["supports_web_search"] is True + + snapshot_cost = StandardBuiltInToolCostTracking.get_cost_for_web_search( + web_search_options=web_search_options, model_info=snapshot_info + ) + alias_cost = StandardBuiltInToolCostTracking.get_cost_for_web_search( + web_search_options=web_search_options, model_info=alias_info + ) + + assert snapshot_cost == alias_cost == 0.025 + + +def test_gpt_4o_mini_web_search_price_matches_in_both_cost_maps(): + """The bundled backup and canonical file are both served to deployments, so both must carry the price.""" + repo_root = Path(__file__).parents[4] + cost_maps = tuple( + json.loads((repo_root / path).read_text(encoding="utf-8")) + for path in ( + "model_prices_and_context_window.json", + "litellm/model_prices_and_context_window_backup.json", + ) + ) + canonical, backup = cost_maps + expected_search_price = { + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025, + "search_context_size_high": 0.025, + } + for model_name in ("gpt-4o-mini", "gpt-4o-mini-2024-07-18"): + canonical_entry = canonical[model_name] + backup_entry = backup[model_name] + assert canonical_entry["search_context_cost_per_query"] == expected_search_price + assert backup_entry["search_context_cost_per_query"] == expected_search_price + assert canonical_entry == backup_entry + + # Note: File search integration test removed due to complex annotation detection logic # The unit tests in test_azure_assistant_cost_tracking.py provide comprehensive coverage diff --git a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py b/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py index eadd87d9c92..08e46b1ffac 100644 --- a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py @@ -322,8 +322,8 @@ class TestModelCostEntry: entry = json.load(f)["vertex_ai/gemini-3.5-transcribe-preview"] assert entry["mode"] == "audio_transcription" assert entry["litellm_provider"] == "vertex_ai" - assert entry["input_cost_per_audio_token"] == pytest.approx(2.5e-06) - assert entry["input_cost_per_token"] == pytest.approx(2.5e-06) + assert entry["input_cost_per_audio_token"] == pytest.approx(2e-06) + assert entry["input_cost_per_token"] == pytest.approx(2e-06) assert entry["output_cost_per_token"] == pytest.approx(1.2e-05) assert entry["supported_endpoints"] == ["/v1/audio/transcriptions"] From 2823d09e1bcfc99cee126583f3552e5d5b32dac5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:30:05 -0700 Subject: [PATCH 108/411] refactor(policy_engine): type the request-state parameter of the retrieval hook --- litellm/proxy/policy_engine/response_retrieval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/policy_engine/response_retrieval.py b/litellm/proxy/policy_engine/response_retrieval.py index 54b9acf6150..0aa9579ca71 100644 --- a/litellm/proxy/policy_engine/response_retrieval.py +++ b/litellm/proxy/policy_engine/response_retrieval.py @@ -71,7 +71,7 @@ def _post_call_pipelines_for_context(context: PolicyMatchContext) -> tuple[Polic def attach_post_call_pipelines_to_retrieval( - data: dict, # mutable-ok: the proxy's request-state dict, written in place like every other policy engine hook + data: dict[str, object], # mutable-ok: request-state dict the policy engine hooks all write in place user_api_key_dict: "UserAPIKeyAuth", llm_router: "Router | None", ) -> None: From 3f30c05493091f62d1edcfb0af58aa33a370a335 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:39:39 -0700 Subject: [PATCH 109/411] fix(guardrails): skip only stream-gated pipeline guardrails in the deferred post-call pass --- litellm/proxy/common_request_processing.py | 6 +- litellm/proxy/utils.py | 37 +++++++++-- .../test_proxy_logging_hook_detection.py | 63 +++++++++++++++++++ 3 files changed, 98 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7e85b66682c..2a6eb0a16ff 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3343,15 +3343,15 @@ class ProxyBaseLLMRequestProcessing: from litellm.proxy.proxy_server import llm_router as _global_llm_router from litellm.proxy.utils import ( _check_and_merge_model_level_guardrails, - pipeline_managed_guardrail_names, + stream_gated_guardrail_names, ) guardrail_data = _check_and_merge_model_level_guardrails(data=captured_data, llm_router=_global_llm_router) - pipeline_managed: Final = pipeline_managed_guardrail_names(captured_data, "post_call") + stream_gated: Final = stream_gated_guardrail_names(captured_data, captured_user_api_key_dict) for cb in litellm.callbacks: if not isinstance(cb, CustomGuardrail): continue - if cb.guardrail_name in pipeline_managed: + if cb.guardrail_name in stream_gated: continue if not cb.should_run_guardrail( data=guardrail_data, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bbec1b842c4..106fab7af8a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -538,12 +538,16 @@ def _warn_background_skips_post_call_pipelines(data: Mapping[str, object]) -> No ) -def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: - unsupported: Final = tuple( +def _pipeline_unsupported_streaming_guardrails(pipeline: "GuardrailPipeline") -> tuple[str, ...]: + return tuple( dict.fromkeys( step.guardrail for step in pipeline.steps if not _pipeline_step_supports_streaming(step.guardrail) ) ) + + +def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: + unsupported: Final = _pipeline_unsupported_streaming_guardrails(pipeline) if not unsupported: return True verbose_proxy_logger.warning( @@ -573,13 +577,12 @@ def _streamable_post_call_pipelines( post_call_pipelines: Final = _post_call_pipelines(request_data) if not post_call_pipelines: return () - route: Final = user_api_key_dict.request_route - if route and resolve_endpoint_translation(user_api_key_dict, None) is None: + if not _route_has_endpoint_translation(user_api_key_dict): verbose_proxy_logger.warning( "Policies with post_call guardrail pipelines cannot scan streaming responses on route %s yet " "(no endpoint guardrail translation); the stream skips the pipelines and their guardrails run " "on their own: %s", - route, + user_api_key_dict.request_route, ", ".join(policy_name for policy_name, _pipeline in post_call_pipelines), ) return () @@ -590,6 +593,30 @@ def _streamable_post_call_pipelines( ) +def _route_has_endpoint_translation(user_api_key_dict: UserAPIKeyAuth) -> bool: + return not user_api_key_dict.request_route or resolve_endpoint_translation(user_api_key_dict, None) is not None + + +def stream_gated_guardrail_names( + request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth +) -> frozenset[str]: + """ + The guardrails whose post_call pipelines gate a streaming response on this + route: the selection ``_streamable_post_call_pipelines`` makes, without its + warnings, so the post-call pass deferred to the end of the stream skips + exactly the guardrails the pipelines already ran and no others. + """ + if not _route_has_endpoint_translation(user_api_key_dict): + return frozenset() + return _pipeline_step_guardrail_names( + tuple( + (policy_name, pipeline) + for policy_name, pipeline in _post_call_pipelines(request_data) + if not _pipeline_unsupported_streaming_guardrails(pipeline) + ) + ) + + def _prompt_block_text(block: object) -> str: if isinstance(block, str): return block diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index f4165f98ba5..10de7e7fc3e 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -697,6 +697,69 @@ async def test_deferred_stream_guardrails_skip_pipeline_managed_native_hook(monk assert pipeline_managed.native_hooks_ran == [] +@pytest.mark.asyncio +async def test_deferred_stream_guardrails_run_native_hook_whose_pipeline_could_not_stream(monkeypatch): + """A pipeline step with neither streaming interface keeps the whole pipeline off the + stream, so the deferred audit is the only place the opted-out guardrail's own hook + still runs, the way it did before pipelines ran on streams.""" + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + from litellm.types.utils import Choices, Message, ModelResponse + + class NeitherHookGuardrail(CustomGuardrail): + pass + + pipeline_managed = _KeepsNativeHooks(event_hook=GuardrailEventHooks.post_call, default_on=True) + neither = NeitherHookGuardrail(guardrail_name="gr-neither", event_hook=GuardrailEventHooks.post_call) + monkeypatch.setattr(litellm, "callbacks", [pipeline_managed, neither]) + pipeline = GuardrailPipeline( + mode="post_call", + steps=[ + PipelineStep(guardrail="keeps_native", on_fail="next"), + PipelineStep(guardrail="gr-neither", on_fail="block"), + ], + ) + + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={ + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"_guardrail_pipelines": [("response-governance", pipeline)]}, + }, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/chat/completions"), + captured_logging_obj=_streaming_logging_obj(), + assembled_response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="hello"))]), + cache_hit=False, + ) + + assert pipeline_managed.native_hooks_ran == ["post_call"] + + +@pytest.mark.asyncio +async def test_deferred_stream_guardrails_run_native_hook_on_route_without_translation(monkeypatch): + """A route with no endpoint guardrail translation cannot gate the stream through its + pipelines, so the deferred audit still owes the opted-out guardrail its own hook.""" + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + from litellm.types.utils import Choices, Message, ModelResponse + + pipeline_managed = _KeepsNativeHooks(event_hook=GuardrailEventHooks.post_call, default_on=True) + monkeypatch.setattr(litellm, "callbacks", [pipeline_managed]) + pipeline = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="keeps_native", on_fail="block")]) + + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={ + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"_guardrail_pipelines": [("response-governance", pipeline)]}, + }, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/custom/stream"), + captured_logging_obj=_streaming_logging_obj(), + assembled_response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="hello"))]), + cache_hit=False, + ) + + assert pipeline_managed.native_hooks_ran == ["post_call"] + + @pytest.mark.asyncio async def test_realtime_guardrails_skip_opted_out_guardrail(monkeypatch): """The realtime path calls apply_guardrail directly, so the opt-out has to be From cf0488316e23b3f9859691d77043e49eb97f6eb6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:40:28 -0700 Subject: [PATCH 110/411] fix(spend-tracking): take each unresolved key's newest spend row and drop docstrings --- .../common_daily_activity.py | 4 +- .../spend_tracking/key_metadata_recovery.py | 42 +++++++------------ .../test_common_daily_activity.py | 5 --- .../test_key_metadata_recovery.py | 5 --- 4 files changed, 16 insertions(+), 40 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index c6c5aad5926..a8aef30107c 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -462,9 +462,7 @@ async def get_api_key_metadata( This ensures that key_alias and team_id are preserved in historical activity logs even after a key is deleted or regenerated. Also recovers aliases for api_key - values that were double-hashed by the v1.99 spend-log provenance gate, and, when - spend_logs_window is given, for keys never written to either token table (CLI - session tokens) from the spend-log rows those requests wrote in that window. + values that were double-hashed by the v1.99 spend-log provenance gate. """ key_records: Sequence[PrismaVerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many( where={"token": {"in": list(api_keys)}} diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 81c1f4cf119..e207f226255 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -29,16 +29,21 @@ ORDER BY token, deleted_at DESC """ _SPEND_LOG_ALIAS_SQL: Final = """ -SELECT DISTINCT ON (api_key) - api_key AS digest, - metadata->>'user_api_key_alias' AS key_alias, - COALESCE(NULLIF(team_id, ''), metadata->>'user_api_key_team_id') AS team_id, - COALESCE(NULLIF("user", ''), metadata->>'user_api_key_user_id') AS user_id -FROM "LiteLLM_SpendLogs" -WHERE api_key = ANY($1::text[]) - AND "startTime" >= $2::timestamp - AND "startTime" < $3::timestamp -ORDER BY api_key, (metadata->>'user_api_key_alias') IS NULL, "startTime" DESC +SELECT newest.digest, newest.key_alias, newest.team_id, newest.user_id +FROM unnest($1::text[]) AS missing(digest) +CROSS JOIN LATERAL ( + SELECT + api_key AS digest, + metadata->>'user_api_key_alias' AS key_alias, + COALESCE(NULLIF(team_id, ''), metadata->>'user_api_key_team_id') AS team_id, + COALESCE(NULLIF("user", ''), metadata->>'user_api_key_user_id') AS user_id + FROM "LiteLLM_SpendLogs" + WHERE api_key = missing.digest + AND "startTime" >= $2::timestamp + AND "startTime" < $3::timestamp + ORDER BY "startTime" DESC + LIMIT 1 +) AS newest """ @@ -152,14 +157,6 @@ async def recover_double_hashed_key_metadata( prisma_client: PrismaClient, missing_keys: AbstractSet[str], ) -> Mapping[str, KeyMetadataDict]: - """ - Recover key_alias/team_id/user_id for DailyUserSpend.api_key values that - were double-hashed by the v1.99 spend-log provenance gate. - - Those rows store hash(VerificationToken.token) instead of the token, so the - exact join misses. Postgres hashes the token column itself, one pass over - active keys and one over deleted keys, so no key row crosses the wire. - """ sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) if not sha_missing: return _EMPTY_KEY_METADATA @@ -187,15 +184,6 @@ async def recover_key_metadata_from_spend_logs( missing_keys: AbstractSet[str], window: tuple[datetime, datetime], ) -> Mapping[str, KeyMetadataDict]: - """ - Recover key_alias/team_id/user_id for hashed api_key values absent from both - verification-token tables, e.g. in-memory CLI session tokens that never get a - token row. Their owner is written to LiteLLM_SpendLogs metadata at request - time under the same hashed api_key, so it is the only surviving source. Only - sha256 digests are looked up, matching the reverse-hash recovery gate, since - every current api_key value in spend logs is a token hash. The [start, end) - bound keeps the lookup on the startTime index instead of scanning the table. - """ sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) if not sha_missing: return _EMPTY_KEY_METADATA diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 4d3dc320d8f..4c8be415204 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -2109,11 +2109,6 @@ async def test_get_daily_activity_aggregated_with_entity_breakdown(): @pytest.mark.asyncio async def test_get_api_key_metadata_resolves_session_key_via_spend_log_window(): - """ - A CLI session token has no verification-token row, so the active/deleted lookups - and reverse-hash all miss. Given a spend-log window, its alias and owner are - recovered from the spend-log metadata and its email is filled from the user table. - """ from litellm.proxy.utils import hash_token session_digest = hash_token("cli-session-user-42") diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 9e42f017106..05a90137e3b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -233,11 +233,6 @@ async def test_fill_missing_api_key_aliases_skips_named_keys_that_have_no_email( @pytest.mark.asyncio async def test_recover_key_metadata_from_spend_logs_resolves_session_token_from_metadata(): - """ - CLI session tokens never get a verification-token row, so both the exact join and - the reverse-hash lookup miss them. Their owner survives only in the spend-log - metadata written at request time, keyed by the same hashed api_key. - """ session_digest = hash_token("cli-session-repro-user-6852") window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) mock_prisma = MagicMock() From 8eb6f6d8350dd0ef7f2e5a61b042213e0b6bd319 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 8 Sep 2026 19:43:32 +0000 Subject: [PATCH 111/411] test(cost_calc): type web search test params and drop docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/test_tool_call_cost_tracking.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 38bf8c544c7..f319413045f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -874,9 +874,8 @@ def test_dated_search_preview_entries_carry_search_pricing(local_model_cost_map) ], ) def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias( - web_search_options, local_model_cost_map -): - """Snapshot and alias must bill web search identically: OpenAI lists preview search at $25 per 1k calls.""" + web_search_options: WebSearchOptions | None, local_model_cost_map: None +) -> None: alias_info = litellm.get_model_info("gpt-4o-mini") snapshot_info = litellm.get_model_info("gpt-4o-mini-2024-07-18") @@ -894,7 +893,6 @@ def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias( def test_gpt_4o_mini_web_search_price_matches_in_both_cost_maps(): - """The bundled backup and canonical file are both served to deployments, so both must carry the price.""" repo_root = Path(__file__).parents[4] cost_maps = tuple( json.loads((repo_root / path).read_text(encoding="utf-8")) From 5e056a264ecff102a72ee36fed333e5e2ecd9c37 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:45:05 -0700 Subject: [PATCH 112/411] refactor(azure): move the passthrough deployment-segment helpers under llms --- .../llms/azure/passthrough/transformation.py | 23 +++++++++++- litellm/proxy/auth/auth_utils.py | 8 +++-- .../llm_passthrough_endpoints.py | 26 ++++---------- .../test_azure_passthrough_transformation.py | 35 ++++++++++++++++++- .../test_llm_pass_through_endpoints.py | 33 +++-------------- 5 files changed, 71 insertions(+), 54 deletions(-) diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index dc71c53e126..51a409c61da 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -1,4 +1,5 @@ -from collections.abc import Mapping, Sequence +import re +from collections.abc import Callable, Collection, Mapping, Sequence from typing import TYPE_CHECKING, Final, Optional import httpx @@ -65,6 +66,26 @@ def logged_responses_stream(all_chunks: Sequence[str], logging_obj: Logging) -> return terminal_event +AZURE_DEPLOYMENT_SEGMENT: Final = re.compile(r"(? str | None: + parts: Final = endpoint.split("/") + if len(parts) < 2: + return None + return next((part for part in parts if part in router_models), None) + + +def foreign_azure_deployment( + endpoint: str, model_group: str, served_models: Callable[[], Collection[str]] +) -> str | None: + match: Final = AZURE_DEPLOYMENT_SEGMENT.search(endpoint) + if match is None: + return None + deployment: Final = match.group(1) + return None if deployment == model_group or deployment in served_models() else deployment + + def without_api_version(api_base: str) -> str: url: Final = httpx.URL(api_base) kept_params: Final = tuple((key, value) for key, value in url.params.multi_items() if key != "api-version") diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index bb6ebddf77a..aa77835e858 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.url_utils import ( provider_url_destination_candidates, validate_url, ) +from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint from litellm.proxy._types import * from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata from litellm.types.passthrough_endpoints.pass_through_endpoints import ( @@ -2011,9 +2012,10 @@ def get_model_from_request( def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str | None: - from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import azure_router_model_in_endpoint - - return azure_router_model_in_endpoint(re.sub(r"^/azure(?:_ai)?/", "", route, flags=re.IGNORECASE), llm_router) + if llm_router is None: + return None + endpoint: Final = re.sub(r"^/azure(?:_ai)?/", "", route, flags=re.IGNORECASE) + return azure_router_model_in_endpoint(endpoint, frozenset(llm_router.get_model_names())) def _model_from_bedrock_route(route: str) -> str | None: diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 47b3c56e66a..66987042962 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -33,6 +33,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.azure.passthrough.transformation import foreign_azure_deployment from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse @@ -121,16 +122,6 @@ def is_passthrough_request_using_router_model(request_body: dict, llm_router: li return False -def azure_router_model_in_endpoint(endpoint: str, llm_router: litellm.Router | None) -> str | None: - parts: Final = endpoint.split("/") - if len(parts) < 2: - return None - return next((part for part in parts if is_known_model(part, llm_router)), None) - - -AZURE_DEPLOYMENT_SEGMENT: Final = re.compile(r"(? str: model: Final = litellm_params.get("model", "") try: @@ -139,17 +130,10 @@ def _deployment_model_name(litellm_params: LiteLLMParamsTypedDict) -> str: return model -def foreign_azure_deployment(endpoint: str, model_group: str, llm_router: litellm.Router) -> str | None: - match: Final = AZURE_DEPLOYMENT_SEGMENT.search(endpoint) - if match is None: - return None - deployment: Final = match.group(1) - if deployment == model_group: - return None - served: Final = frozenset( +def _models_served_by_group(llm_router: litellm.Router, model_group: str) -> frozenset[str]: + return frozenset( _deployment_model_name(row["litellm_params"]) for row in llm_router.get_model_list(model_name=model_group) or () ) - return None if deployment in served else deployment def is_passthrough_request_streaming(request_body: object) -> bool: @@ -1555,7 +1539,9 @@ async def _relay_azure_router_model( is_streaming_request: bool, user_api_key_dict: UserAPIKeyAuth, ) -> Response: - foreign_deployment: Final = foreign_azure_deployment(endpoint, model, llm_router) + foreign_deployment: Final = foreign_azure_deployment( + endpoint, model, lambda: _models_served_by_group(llm_router, model) + ) if foreign_deployment is not None: raise HTTPException( status_code=400, diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index a3f5c415417..69fdd59a748 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -8,7 +8,11 @@ import pytest import litellm from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound -from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig +from litellm.llms.azure.passthrough.transformation import ( + AzurePassthroughConfig, + azure_router_model_in_endpoint, + foreign_azure_deployment, +) from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse from litellm.types.utils import EmbeddingResponse, ModelResponse @@ -444,3 +448,32 @@ def test_azure_passthrough_is_streaming_request_reads_the_stream_flag(request_da ) is expected ) + + +@pytest.mark.parametrize( + "endpoint, expected", + [ + ("gpt/openai/deployments/gpt/chat/completions", None), + ("openai/deployments/gpt/chat/completions", None), + ("gpt/openai/deployments/gpt-5.4-mini/chat/completions", None), + ("gpt/models/chat/completions", None), + ("gpt/openai/deployments/gpt-5.4/chat/completions", "gpt-5.4"), + ("gpt/openai/deployments/other-group/chat/completions", "other-group"), + ("openai/deployments/victim/gpt/chat/completions", "victim"), + ], +) +def test_foreign_azure_deployment_names_a_segment_outside_the_group(endpoint, expected): + assert foreign_azure_deployment(endpoint, "gpt", lambda: frozenset({"gpt-5.4-mini"})) == expected + + +@pytest.mark.parametrize( + "endpoint, expected", + [ + ("other-group/openai/deployments/other-group/chat/completions", "other-group"), + ("openai/deployments/gpt/chat/completions", "gpt"), + ("openai/deployments/my-azure-deployment/chat/completions", None), + ("gpt", None), + ], +) +def test_azure_router_model_in_endpoint_picks_the_first_router_model_segment(endpoint, expected): + assert azure_router_model_in_endpoint(endpoint, frozenset({"gpt", "other-group"})) == expected diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index fe4400df704..d1f9e5c4c1d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -5368,36 +5368,11 @@ class TestAzureRelayDeploymentSegment: """A key allowed one model group must not reach another deployment by naming it in the ``openai/deployments/`` segment while the group segment picks the credential.""" - @pytest.mark.parametrize( - "endpoint, expected", - [ - ("gpt/openai/deployments/gpt/chat/completions", None), - ("openai/deployments/gpt/chat/completions", None), - ("gpt/openai/deployments/gpt-5.4-mini/chat/completions", None), - ("gpt/models/chat/completions", None), - ("gpt/openai/deployments/gpt-5.4/chat/completions", "gpt-5.4"), - ("gpt/openai/deployments/other-group/chat/completions", "other-group"), - ("openai/deployments/victim/gpt/chat/completions", "victim"), - ], - ) - def test_foreign_azure_deployment_names_a_segment_outside_the_group(self, endpoint, expected): - from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import foreign_azure_deployment + def test_models_served_by_group_resolves_each_deployment_to_its_model_name(self): + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import _models_served_by_group - assert foreign_azure_deployment(endpoint, "gpt", _AzureGroupRouter([])) == expected - - @pytest.mark.parametrize( - "endpoint, expected", - [ - ("other-group/openai/deployments/other-group/chat/completions", "other-group"), - ("openai/deployments/gpt/chat/completions", "gpt"), - ("openai/deployments/my-azure-deployment/chat/completions", None), - ("gpt", None), - ], - ) - def test_azure_router_model_in_endpoint_matches_the_relay_decision(self, endpoint, expected): - from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import azure_router_model_in_endpoint - - assert azure_router_model_in_endpoint(endpoint, _AzureGroupRouter([])) == expected + assert _models_served_by_group(_AzureGroupRouter([]), "gpt") == frozenset({"gpt-5.4-mini"}) + assert _models_served_by_group(_AzureGroupRouter([]), "missing-group") == frozenset() def _install(self, monkeypatch, body: dict) -> list[dict]: import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep From 91091fd93e7b7be6576be26e1ef1216391af667e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:59:58 -0700 Subject: [PATCH 113/411] fix: withdraw policy header claims while a background response is pending and log ungoverned retrievals --- .../proxy/policy_engine/response_retrieval.py | 31 ++++-- litellm/proxy/utils.py | 62 +++++++++++- .../policy_engine/test_response_retrieval.py | 36 +++++-- .../proxy_logging/test_guardrail_pipeline.py | 97 +++++++++++++++++++ 4 files changed, 205 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/policy_engine/response_retrieval.py b/litellm/proxy/policy_engine/response_retrieval.py index 0aa9579ca71..8fc75e40070 100644 --- a/litellm/proxy/policy_engine/response_retrieval.py +++ b/litellm/proxy/policy_engine/response_retrieval.py @@ -1,6 +1,7 @@ from collections.abc import Mapping +from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import TypeAdapter @@ -29,14 +30,23 @@ PolicyPipelines: TypeAlias = tuple[tuple[str, GuardrailPipeline], ...] _POLICY_PIPELINES_ADAPTER: Final = TypeAdapter(PolicyPipelines) -def _model_group_for_response_id(response_id: object, llm_router: "Router | None") -> str | None: - if llm_router is None or not isinstance(response_id, str): - return None - model_id: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id) +@dataclass(frozen=True, slots=True) +class UngovernedRetrieval: + reason: Literal["no router", "response id names no deployment", "deployment no longer in the router"] + + +def _model_group_for_response_id(response_id: object, llm_router: "Router | None") -> str | UngovernedRetrieval: + if llm_router is None: + return UngovernedRetrieval("no router") + model_id: Final = ( + ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id) if isinstance(response_id, str) else None + ) if model_id is None: - return None + return UngovernedRetrieval("response id names no deployment") deployment: Final = llm_router.get_deployment(model_id) - return deployment.model_name if deployment is not None else None + if deployment is None: + return UngovernedRetrieval("deployment no longer in the router") + return deployment.model_name def _retrieval_context( @@ -78,7 +88,12 @@ def attach_post_call_pipelines_to_retrieval( if not get_policy_registry().is_initialized(): return model_group: Final = _model_group_for_response_id(data.get("response_id"), llm_router) - if model_group is None: + if isinstance(model_group, UngovernedRetrieval): + verbose_proxy_logger.warning( + "Policy engine: background response %s is retrieved without its post_call policy pipelines (%s)", + data.get("response_id"), + model_group.reason, + ) return context: Final = _retrieval_context(data, user_api_key_dict, model_group) post_call_pipelines, policy_sources = _post_call_pipelines_for_context(context) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3d03d3858ee..1972d92b610 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -93,6 +93,7 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert from litellm.litellm_core_utils.core_helpers import ( coerce_token_limit, + get_or_create_metadata_bucket, independent_snapshot, is_expected_client_error, ) @@ -156,6 +157,8 @@ from litellm.proxy.hooks.sensitive_data_routing import ( from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_guardrails_from_auth_metadata from litellm.proxy.management_helpers.key_settings_audit import with_settings_updated_at from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor +from litellm.proxy.policy_engine.policy_registry import get_policy_registry +from litellm.proxy.policy_engine.policy_resolver import PolicyResolver from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.table_repositories import ( @@ -533,16 +536,65 @@ def _is_pending_background_response(response: LLMResponseTypes) -> bool: return isinstance(response, ResponsesAPIResponse) and response.status in _PENDING_BACKGROUND_RESPONSE_STATUSES -def _log_deferred_post_call_pipelines(data: Mapping[str, object], response: ResponsesAPIResponse) -> None: - policy_names: Final = tuple(policy_name for policy_name, _pipeline in _post_call_pipelines(data)) - if not policy_names: +def _guardrails_outside_pipeline(policy_name: str, pipeline: "GuardrailPipeline") -> frozenset[str]: + resolved: Final = PolicyResolver.resolve_policy_guardrails( + policy_name=policy_name, policies=get_policy_registry().get_all_policies() + ) + return frozenset(resolved.guardrails) - frozenset(step.guardrail for step in pipeline.steps) + + +def _without_names( + bucket: dict, # mutable-ok: the applied_* header slots live in the request-state dict every hook writes in place + slot: str, + names: frozenset[str], +) -> None: + claimed: Final = bucket.get(slot) + if not isinstance(claimed, list): + return + remaining: Final = [name for name in claimed if name not in names] + if remaining: + bucket[slot] = remaining + else: + bucket.pop(slot) + + +def _withdraw_deferred_claims( + data: dict, # mutable-ok: same request-payload shape as post_call_success_hook's data + deferred: Sequence[tuple[str, "GuardrailPipeline"]], +) -> None: + outside_by_policy: Final = MappingProxyType( + {policy_name: _guardrails_outside_pipeline(policy_name, pipeline) for policy_name, pipeline in deferred} + ) + running_elsewhere: Final = _pipeline_managed_guardrail_names(data, "pre_call").union(*outside_by_policy.values()) + withdrawn_policies: Final = frozenset(name for name, outside in outside_by_policy.items() if not outside) + withdrawn_guardrails: Final = _pipeline_step_guardrail_names(deferred) - running_elsewhere + _, bucket = get_or_create_metadata_bucket(data) + _without_names(bucket, "applied_policies", withdrawn_policies) + _without_names(bucket, "applied_guardrails", withdrawn_guardrails) + sources: Final = bucket.get("policy_sources") + if not isinstance(sources, dict): + return + remaining_sources: Final = {name: reason for name, reason in sources.items() if name not in withdrawn_policies} + if remaining_sources: + bucket["policy_sources"] = remaining_sources + else: + bucket.pop("policy_sources") + + +def _defer_post_call_pipelines( + data: dict, # mutable-ok: same request-payload shape as post_call_success_hook's data + response: ResponsesAPIResponse, +) -> None: + deferred: Final = _post_call_pipelines(data) + if not deferred: return verbose_proxy_logger.debug( "Post_call guardrail pipelines wait for background response %s (status=%s) to be retrieved complete: %s", response.id, response.status, - ", ".join(policy_names), + ", ".join(policy_name for policy_name, _pipeline in deferred), ) + _withdraw_deferred_claims(data, deferred) def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: @@ -2939,7 +2991,7 @@ class ProxyLogging: response: LLMResponseTypes, ) -> LLMResponseTypes | None: if _is_pending_background_response(response): - _log_deferred_post_call_pipelines(data, response) + _defer_post_call_pipelines(data, response) return None _, pipeline_response = await self._maybe_execute_pipelines( data=data, diff --git a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py index f7f29d6b46e..8c00b18eb63 100644 --- a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py +++ b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py @@ -1,3 +1,5 @@ +import logging + import pytest from litellm.proxy._types import UserAPIKeyAuth @@ -128,30 +130,48 @@ def test_already_attached_policy_is_not_attached_twice(policy_engine): assert data["litellm_metadata"]["applied_policies"] == ["response-governance"] +def _ungoverned_retrieval_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and "retrieved without its post_call policy pipelines" in record.getMessage() + ] + + @pytest.mark.parametrize( - "response_id", - ["resp_plain_upstream_id", _encoded_response_id("deployment-missing-from-router"), None], + ("response_id", "reason"), + [ + ("resp_plain_upstream_id", "response id names no deployment"), + (_encoded_response_id("deployment-missing-from-router"), "deployment no longer in the router"), + (None, "response id names no deployment"), + ], ) -def test_unresolvable_response_id_attaches_nothing(policy_engine, response_id): +def test_unresolvable_response_id_attaches_nothing_and_warns(policy_engine, caplog, response_id, reason): data = {"response_id": response_id, "litellm_metadata": {}} - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) assert data == {"response_id": response_id, "litellm_metadata": {}} + assert [message.endswith(f"({reason})") for message in _ungoverned_retrieval_warnings(caplog)] == [True] -def test_without_a_router_attaches_nothing(policy_engine): +def test_without_a_router_attaches_nothing_and_warns(policy_engine, caplog): data = _retrieval_data(GOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) assert data == _retrieval_data(GOVERNED_MODEL_ID) + assert [message.endswith("(no router)") for message in _ungoverned_retrieval_warnings(caplog)] == [True] -def test_without_policy_engine_attaches_nothing(): +def test_without_policy_engine_attaches_nothing_quietly(caplog): get_policy_registry().clear() data = _retrieval_data(GOVERNED_MODEL_ID) - attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=None) assert data == _retrieval_data(GOVERNED_MODEL_ID) + assert _ungoverned_retrieval_warnings(caplog) == [] diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 7d2c232ad97..8da07f39fd8 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -1481,6 +1481,103 @@ async def test_post_call_success_hook_runs_pipeline_on_retrieved_background_resp assert seen["response"] is response +def _output_passing_callbacks() -> list[CustomGuardrail]: + class OutputPassingGuardrail(CustomGuardrail): + async def async_post_call_success_hook(self, data, user_api_key_dict, response): + return response + + return [OutputPassingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)] + + +def _claimed_post_call_pipeline_data(*policy_names: str, extra_guardrails: dict[str, list[str]] | None = None): + from litellm.proxy.policy_engine.policy_registry import get_policy_registry + + step = {"guardrail": "gr-post", "on_pass": "allow", "on_fail": "block"} + get_policy_registry().load_policies( + { + policy_name: { + "guardrails": {"add": ["gr-post", *(extra_guardrails or {}).get(policy_name, [])]}, + "pipeline": {"mode": "post_call", "steps": [step]}, + } + for policy_name in policy_names + } + ) + pipeline = GuardrailPipeline(mode="post_call", steps=[PipelineStep(**step)]) + return { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "metadata": { + "_guardrail_pipelines": [(policy_name, pipeline) for policy_name in policy_names], + "_pipeline_managed_guardrails": {"gr-post"}, + "applied_policies": list(policy_names), + "applied_guardrails": ["gr-post", *(g for gs in (extra_guardrails or {}).values() for g in gs)], + "policy_sources": {policy_name: "model:m" for policy_name in policy_names}, + }, + } + + +@pytest.fixture +def clear_policy_registry(): + from litellm.proxy.policy_engine.policy_registry import get_policy_registry + + yield + get_policy_registry().clear() + + +@pytest.mark.asyncio +async def test_pending_background_response_withdraws_the_deferred_policy_claims( + proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry +): + monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _claimed_post_call_pipeline_data("response-governance") + + out = await proxy_logging.post_call_success_hook( + data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth() + ) + + assert out.status == "queued" + assert "applied_policies" not in data["metadata"] + assert "policy_sources" not in data["metadata"] + assert "applied_guardrails" not in data["metadata"] + + +@pytest.mark.asyncio +async def test_pending_background_response_keeps_the_claim_of_a_policy_that_runs_outside_its_pipeline( + proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry +): + monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _claimed_post_call_pipeline_data( + "input-and-output-governance", "response-governance", extra_guardrails={"input-and-output-governance": ["gr-pre"]} + ) + + await proxy_logging.post_call_success_hook( + data=data, response=_background_response("in_progress"), user_api_key_dict=make_user_api_key_auth() + ) + + assert data["metadata"]["applied_policies"] == ["input-and-output-governance"] + assert data["metadata"]["applied_guardrails"] == ["gr-pre"] + assert data["metadata"]["policy_sources"] == {"input-and-output-governance": "model:m"} + + +@pytest.mark.asyncio +async def test_retrieved_background_response_keeps_the_policy_claim_once_its_pipeline_ran( + proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry +): + monkeypatch.setattr(litellm, "callbacks", _output_passing_callbacks()) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _claimed_post_call_pipeline_data("response-governance") + + await proxy_logging.post_call_success_hook( + data=data, response=_background_response("completed", text="fine"), user_api_key_dict=make_user_api_key_auth() + ) + + assert data["metadata"]["applied_policies"] == ["response-governance"] + assert data["metadata"]["policy_sources"] == {"response-governance": "model:m"} + assert data["metadata"]["applied_guardrails"] == ["gr-post"] + + @pytest.mark.asyncio async def test_pre_call_hook_stays_quiet_on_background_request_with_post_call_pipeline( proxy_logging, make_user_api_key_auth, monkeypatch, caplog From 550e733bf35ac862970cfebeed872e45f539bc7d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:09:43 -0700 Subject: [PATCH 114/411] perf(spend-tracking): recover key identity in one bounded spend-log pass with a per-worker cache The spend-log lookup for permanently unresolvable digests is back to a single DISTINCT ON scan over the requested window, keeping only rows that carry an alias, user, or team so a newer nameless row cannot hide an older named one. Results and misses are cached per worker for ten minutes keyed by digest and window, failed queries are not cached, and JWT rows keyed hashed-jwt- now pass the digest gate. Tests cover the JWT gate, cache reuse and partial misses, window changes, error handling, the with-window guard, and the daily activity wiring --- litellm/constants.py | 2 + .../spend_tracking/key_metadata_recovery.py | 105 +++++++++++++----- .../test_common_daily_activity.py | 74 +++++++++++- .../test_key_metadata_recovery.py | 99 ++++++++++++++++- 4 files changed, 248 insertions(+), 32 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 8ef9523a60a..c1896573eb5 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1763,6 +1763,8 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)) +SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = int(os.getenv("SPEND_LOG_KEY_METADATA_CACHE_TTL", "600")) +SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = int(os.getenv("SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS", "10000")) # Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated # callers from forcing a DB query per request for unknown names, while bounding # staleness so a transient DB error (which surfaces as an empty list) cannot diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index e207f226255..7212d8b6acb 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -8,6 +8,8 @@ from pydantic import BaseModel, TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS, SPEND_LOG_KEY_METADATA_CACHE_TTL from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash from litellm.proxy.utils import PrismaClient from litellm.repositories.user_repository import UserRepository @@ -29,23 +31,27 @@ ORDER BY token, deleted_at DESC """ _SPEND_LOG_ALIAS_SQL: Final = """ -SELECT newest.digest, newest.key_alias, newest.team_id, newest.user_id -FROM unnest($1::text[]) AS missing(digest) -CROSS JOIN LATERAL ( - SELECT - api_key AS digest, - metadata->>'user_api_key_alias' AS key_alias, - COALESCE(NULLIF(team_id, ''), metadata->>'user_api_key_team_id') AS team_id, - COALESCE(NULLIF("user", ''), metadata->>'user_api_key_user_id') AS user_id - FROM "LiteLLM_SpendLogs" - WHERE api_key = missing.digest - AND "startTime" >= $2::timestamp - AND "startTime" < $3::timestamp - ORDER BY "startTime" DESC - LIMIT 1 -) AS newest +SELECT DISTINCT ON (api_key) + api_key AS digest, + metadata->>'user_api_key_alias' AS key_alias, + COALESCE(NULLIF(team_id, ''), metadata->>'user_api_key_team_id') AS team_id, + COALESCE(NULLIF("user", ''), metadata->>'user_api_key_user_id') AS user_id +FROM "LiteLLM_SpendLogs" +WHERE api_key = ANY($1::text[]) + AND "startTime" >= $2::timestamp + AND "startTime" < $3::timestamp + AND COALESCE( + metadata->>'user_api_key_alias', + NULLIF("user", ''), + metadata->>'user_api_key_user_id', + NULLIF(team_id, ''), + metadata->>'user_api_key_team_id' + ) IS NOT NULL +ORDER BY api_key, "startTime" DESC """ +_HASHED_JWT_PREFIX: Final = "hashed-jwt-" + class KeyMetadataDict(TypedDict, total=False): key_alias: ReadOnly[str | None] @@ -62,6 +68,11 @@ class _TokenDigestRow(BaseModel): _TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...]) +_CACHED_KEY_METADATA: Final = TypeAdapter(KeyMetadataDict) +_SPEND_LOG_METADATA_CACHE: Final = InMemoryCache( + max_size_in_memory=SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS, + default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL, +) _EMPTY_KEY_METADATA: Final[Mapping[str, KeyMetadataDict]] = MappingProxyType({}) _EMPTY_EMAILS: Final[Mapping[str, str]] = MappingProxyType({}) @@ -179,31 +190,73 @@ async def recover_double_hashed_key_metadata( return MappingProxyType({**from_active, **from_deleted}) -async def recover_key_metadata_from_spend_logs( - prisma_client: PrismaClient, - missing_keys: AbstractSet[str], +def _is_spend_log_digest(key: str) -> bool: + return is_valid_sha256_hash(key.removeprefix(_HASHED_JWT_PREFIX)) + + +def _spend_log_cache_key(digest: str, window: tuple[datetime, datetime]) -> str: + start, end = window + return f"spend_log_key_metadata:{digest}:{start.isoformat()}:{end.isoformat()}" + + +def _cached_spend_log_metadata( + cache: InMemoryCache, + digest: str, window: tuple[datetime, datetime], -) -> Mapping[str, KeyMetadataDict]: - sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key)) - if not sha_missing: - return _EMPTY_KEY_METADATA +) -> KeyMetadataDict | None: + cached: Final[object] = cache.get_cache(_spend_log_cache_key(digest, window)) + return None if cached is None else _CACHED_KEY_METADATA.validate_python(cached) + + +async def _query_spend_log_metadata( + prisma_client: PrismaClient, + digests: AbstractSet[str], + window: tuple[datetime, datetime], +) -> Mapping[str, KeyMetadataDict] | None: start, end = window rows: Final = await _db_or_empty( - lambda: prisma_client.db.query_raw(_SPEND_LOG_ALIAS_SQL, sorted(sha_missing), start, end), + lambda: prisma_client.db.query_raw(_SPEND_LOG_ALIAS_SQL, sorted(digests), start, end), "Failed spend-log alias recovery for %d missing keys: %s", - len(sha_missing), + len(digests), ) if rows is None: - return _EMPTY_KEY_METADATA + return None return MappingProxyType( { row.digest: KeyMetadataDict(key_alias=row.key_alias, team_id=row.team_id, user_id=row.user_id) for row in _TOKEN_DIGEST_ROWS.validate_python(rows) - if row.digest in sha_missing and (row.key_alias or row.user_id or row.team_id) + if row.digest in digests and (row.key_alias or row.user_id or row.team_id) } ) +async def recover_key_metadata_from_spend_logs( + prisma_client: PrismaClient, + missing_keys: AbstractSet[str], + window: tuple[datetime, datetime], + cache: InMemoryCache = _SPEND_LOG_METADATA_CACHE, +) -> Mapping[str, KeyMetadataDict]: + digests: Final = frozenset(key for key in missing_keys if _is_spend_log_digest(key)) + if not digests: + return _EMPTY_KEY_METADATA + cached: Final = MappingProxyType( + { + digest: meta + for digest in digests + for meta in (_cached_spend_log_metadata(cache, digest, window),) + if meta is not None + } + ) + uncached: Final = digests - frozenset(cached) + fresh: Final = await _query_spend_log_metadata(prisma_client, uncached, window) if uncached else _EMPTY_KEY_METADATA + if fresh is not None: + for digest in uncached: + cache.set_cache(_spend_log_cache_key(digest, window), fresh.get(digest, KeyMetadataDict())) + return MappingProxyType( + {digest: meta for digest, meta in (*cached.items(), *(fresh or _EMPTY_KEY_METADATA).items()) if meta} + ) + + def _row_with_recovered_fields( row: Mapping[str, object], recovered: Mapping[str, KeyMetadataDict], diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 4c8be415204..d214ab6b5ac 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -492,7 +492,7 @@ async def test_get_api_key_metadata_recovers_double_hashed_key_via_reverse_hash( @pytest.mark.asyncio async def test_get_api_key_metadata_permanent_miss_never_pages_tokens_or_reads_spend_logs(): - """A dirty key no table can explain costs two digest lookups, never a token page walk or a SpendLogs scan.""" + """Without a spend-log window a dirty key no table can explain costs two digest lookups and never a token page walk.""" from litellm.proxy.utils import hash_token double_hashed = hash_token("b" * 64) @@ -518,6 +518,78 @@ async def test_get_api_key_metadata_permanent_miss_never_pages_tokens_or_reads_s assert all("take" not in call.kwargs and "skip" not in call.kwargs for call in token_lookups) +@pytest.mark.asyncio +async def test_get_api_key_metadata_permanent_miss_with_a_window_reads_spend_logs_once_within_it(): + from litellm.proxy.utils import hash_token + + double_hashed = hash_token("permanent-miss-with-window-6852") + window = (datetime(2024, 1, 1), datetime(2024, 1, 4)) + mock_prisma = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + + result = await get_api_key_metadata(prisma_client=mock_prisma, api_keys={double_hashed}, spend_logs_window=window) + + assert double_hashed not in result + assert mock_prisma.db.query_raw.await_count == 3 + ((_, digests, start, end),) = [ + call.args for call in mock_prisma.db.query_raw.call_args_list if "LiteLLM_SpendLogs" in call.args[0] + ] + assert digests == [double_hashed] + assert (start, end) == window + + +@pytest.mark.asyncio +async def test_get_daily_activity_recovers_a_session_key_alias_from_spend_logs_around_the_page_dates(): + from litellm.proxy.utils import hash_token + + session_digest = hash_token("cli-session-daily-activity-6852") + records = [_daily_user_spend_record(user_id="session-user", api_key=session_digest, spend=1.5)] + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_table = MagicMock() + mock_table.count = AsyncMock(return_value=len(records)) + mock_table.find_many = AsyncMock(return_value=records) + mock_prisma.db.litellm_dailyuserspend = mock_table + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_usertable.find_many = AsyncMock( + return_value=[SimpleNamespace(user_id="session-user", user_email="session@example.com")] + ) + + async def query_raw(sql, *params): + if "LiteLLM_SpendLogs" in sql: + return [{"digest": session_digest, "key_alias": "cli-session-alias", "team_id": None, "user_id": "session-user"}] + return [] + + mock_prisma.db.query_raw = AsyncMock(side_effect=query_raw) + + result = await get_daily_activity( + prisma_client=mock_prisma, + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id=None, + entity_metadata_field=None, + start_date="2024-01-01", + end_date="2024-01-01", + model=None, + api_key=None, + page=1, + page_size=1000, + ) + + key_metadata = result.results[0].breakdown.api_keys[session_digest].metadata + assert key_metadata.key_alias == "cli-session-alias" + assert key_metadata.user_email == "session@example.com" + ((_, digests, start, end),) = [ + call.args for call in mock_prisma.db.query_raw.call_args_list if "LiteLLM_SpendLogs" in call.args[0] + ] + assert digests == [session_digest] + assert (start, end) == (datetime(2023, 12, 31), datetime(2024, 1, 3)) + + def test_key_metadata_includes_recovered_user_email(): from litellm.proxy.management_endpoints.common_daily_activity import _key_metadata diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 05a90137e3b..3d7362b9de0 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from prisma.errors import PrismaError +from litellm.caching.in_memory_cache import InMemoryCache from litellm.proxy.spend_tracking.key_metadata_recovery import ( fill_missing_api_key_aliases, recover_double_hashed_key_metadata, @@ -240,7 +241,7 @@ async def test_recover_key_metadata_from_spend_logs_resolves_session_token_from_ [_digest_row(session_digest, "cli-session-repro-user-6852", None, "repro-user-6852")] ) - result = await recover_key_metadata_from_spend_logs(mock_prisma, {session_digest}, window) + result = await recover_key_metadata_from_spend_logs(mock_prisma, {session_digest}, window, cache=InMemoryCache()) assert result[session_digest]["key_alias"] == "cli-session-repro-user-6852" assert result[session_digest]["user_id"] == "repro-user-6852" @@ -255,7 +256,7 @@ async def test_recover_key_metadata_from_spend_logs_skips_query_when_no_missing_ mock_prisma = MagicMock() mock_prisma.db.query_raw = AsyncMock(return_value=[]) - result = await recover_key_metadata_from_spend_logs(mock_prisma, set(), window) + result = await recover_key_metadata_from_spend_logs(mock_prisma, set(), window, cache=InMemoryCache()) assert result == {} mock_prisma.db.query_raw.assert_not_called() @@ -267,7 +268,9 @@ async def test_recover_key_metadata_from_spend_logs_returns_empty_on_prisma_erro mock_prisma = MagicMock() mock_prisma.db.query_raw = AsyncMock(side_effect=PrismaError("db down")) - result = await recover_key_metadata_from_spend_logs(mock_prisma, {hash_token("cli-session-x")}, window) + result = await recover_key_metadata_from_spend_logs( + mock_prisma, {hash_token("cli-session-x")}, window, cache=InMemoryCache() + ) assert result == {} @@ -287,7 +290,7 @@ async def test_recover_key_metadata_from_spend_logs_ignores_foreign_and_all_null ] ) - result = await recover_key_metadata_from_spend_logs(mock_prisma, {wanted, all_null}, window) + result = await recover_key_metadata_from_spend_logs(mock_prisma, {wanted, all_null}, window, cache=InMemoryCache()) assert set(result) == {wanted} assert result[wanted]["key_alias"] == "kept-alias" @@ -301,8 +304,94 @@ async def test_recover_key_metadata_from_spend_logs_skips_non_sha256_keys(): mock_prisma.db.query_raw = AsyncMock(return_value=[]) result = await recover_key_metadata_from_spend_logs( - mock_prisma, {"cli-session-raw-1798", "key-hash-short"}, window + mock_prisma, {"cli-session-raw-1798", "key-hash-short"}, window, cache=InMemoryCache() ) assert result == {} mock_prisma.db.query_raw.assert_not_called() + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_accepts_hashed_jwt_digests(): + jwt_digest = f"hashed-jwt-{hash_token('jwt-subject-1')}" + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(jwt_digest, None, "team-jwt", "jwt-user")]) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, {jwt_digest}, window, cache=InMemoryCache()) + + assert result[jwt_digest]["team_id"] == "team-jwt" + assert result[jwt_digest]["user_id"] == "jwt-user" + ((_, digests, _, _),) = [call.args for call in mock_prisma.db.query_raw.call_args_list] + assert digests == [jwt_digest] + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_serves_repeat_lookups_from_the_cache(): + found = hash_token("cli-session-found") + unknown = hash_token("cli-session-unknown") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + cache = InMemoryCache() + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(found, "found-alias", None, "owner-1")]) + + first = await recover_key_metadata_from_spend_logs(mock_prisma, {found, unknown}, window, cache=cache) + second = await recover_key_metadata_from_spend_logs(mock_prisma, {found, unknown}, window, cache=cache) + + assert first == second + assert set(first) == {found} + assert first[found]["key_alias"] == "found-alias" + assert mock_prisma.db.query_raw.await_count == 1 + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_only_queries_digests_the_cache_has_not_seen(): + cached_digest = hash_token("cli-session-cached") + new_digest = hash_token("cli-session-new") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + cache = InMemoryCache() + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(cached_digest, "cached-alias", None, None)]) + await recover_key_metadata_from_spend_logs(mock_prisma, {cached_digest}, window, cache=cache) + mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(new_digest, "new-alias", None, None)]) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, {cached_digest, new_digest}, window, cache=cache) + + assert result[cached_digest]["key_alias"] == "cached-alias" + assert result[new_digest]["key_alias"] == "new-alias" + ((_, digests, _, _),) = [call.args for call in mock_prisma.db.query_raw.call_args_list] + assert digests == [new_digest] + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_rescans_when_the_window_changes(): + digest = hash_token("cli-session-windowed") + cache = InMemoryCache() + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs([]) + await recover_key_metadata_from_spend_logs( + mock_prisma, {digest}, (datetime(2026, 9, 1), datetime(2026, 9, 4)), cache=cache + ) + mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(digest, "later-alias", None, None)]) + + result = await recover_key_metadata_from_spend_logs( + mock_prisma, {digest}, (datetime(2026, 9, 7), datetime(2026, 9, 10)), cache=cache + ) + + assert result[digest]["key_alias"] == "later-alias" + assert mock_prisma.db.query_raw.await_count == 1 + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_does_not_cache_a_failed_query(): + digest = hash_token("cli-session-retry") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + cache = InMemoryCache() + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock(side_effect=PrismaError("db down")) + assert await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) == {} + mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(digest, "back-online", None, None)]) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) + + assert result[digest]["key_alias"] == "back-online" From 8ce3fe4fcf5ba43278c4eb7ec09c58df513d36c5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:13:36 -0700 Subject: [PATCH 115/411] style: give the header slot rebuilds and the in-place slot write their lint reasons --- litellm/proxy/utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1972d92b610..e48bd3851fb 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -551,9 +551,11 @@ def _without_names( claimed: Final = bucket.get(slot) if not isinstance(claimed, list): return - remaining: Final = [name for name in claimed if name not in names] + remaining: Final = [ # mutable-ok: the slot stays a list, the shape every applied_* header writer appends to + name for name in claimed if name not in names + ] if remaining: - bucket[slot] = remaining + bucket[slot] = remaining # rebind-ok: the slot lives in the shared request-state dict, rewritten in place else: bucket.pop(slot) @@ -574,7 +576,9 @@ def _withdraw_deferred_claims( sources: Final = bucket.get("policy_sources") if not isinstance(sources, dict): return - remaining_sources: Final = {name: reason for name, reason in sources.items() if name not in withdrawn_policies} + remaining_sources: Final = { # mutable-ok: policy_sources stays a dict, the shape its writer updates in place + name: reason for name, reason in sources.items() if name not in withdrawn_policies + } if remaining_sources: bucket["policy_sources"] = remaining_sources else: From f4aef5a1db67d35f606d09f5e8bfe008181af015 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:19:35 -0700 Subject: [PATCH 116/411] fix(spend-tracking): make the spend-log metadata cache limits plain constants The ten minute TTL and the 10000 item ceiling need no env override, and the documentation env-key check flags any os.getenv read that the docs do not list --- litellm/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c1896573eb5..9654c84c412 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1763,8 +1763,8 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)) -SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = int(os.getenv("SPEND_LOG_KEY_METADATA_CACHE_TTL", "600")) -SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = int(os.getenv("SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS", "10000")) +SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600 +SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000 # Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated # callers from forcing a DB query per request for unknown names, while bounding # staleness so a transient DB error (which surfaces as an empty list) cannot From f59354d09b642bf50d888a82b5fac5b96d34169b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:22:29 -0700 Subject: [PATCH 117/411] refactor(guardrails): patch Anthropic SSE rewrites by field The ended-stream rewriters now describe the one field they change as a frozen _SSEFieldRewrite and a single applier builds the patched event, so the handler adds no mutable-collection constructions over staging's total --- .../chat/guardrail_translation/handler.py | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 2049866b444..cbbccac17f3 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -13,7 +13,7 @@ Pattern Overview: """ import json -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from copy import deepcopy from dataclasses import dataclass from itertools import chain, repeat @@ -161,7 +161,25 @@ class _ToolCallShape: arguments: str -_SSEEventRewriter = Callable[[Mapping[str, object]], Mapping[str, object] | None] +@dataclass(frozen=True, slots=True) +class _SSEFieldRewrite: + """One field of one nested section of a buffered SSE event, rewritten.""" + + section: str + field: str + value: object + + +class _SSEEventRewriter(Protocol): + def __call__(self, event: Mapping[str, object]) -> _SSEFieldRewrite | None: ... + + +def _rewritten_event(event: Mapping[str, object], rewrite_event: _SSEEventRewriter) -> Mapping[str, object]: + rewrite: Final = rewrite_event(event) + section: Final = None if rewrite is None else event.get(rewrite.section) + if rewrite is None or not isinstance(section, Mapping): + return event + return {**event, rewrite.section: {**section, rewrite.field: rewrite.value}} # mutable-ok: json.dumps needs a dict def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...]: @@ -1253,13 +1271,13 @@ class AnthropicMessagesHandler(BaseTranslation): message and content-block framing untouched.""" replacements: Final = chain((rewritten_text,), repeat("")) - def rewrite_text_delta(event: Mapping[str, object]) -> Mapping[str, object] | None: + def rewrite_text_delta(event: Mapping[str, object]) -> _SSEFieldRewrite | None: delta: Final = event.get("delta") if event.get("type") != "content_block_delta" or not isinstance(delta, Mapping): return None if delta.get("type") != "text_delta": return None - return {**event, "delta": {**delta, "text": next(replacements)}} + return _SSEFieldRewrite("delta", "text", next(replacements)) AnthropicMessagesHandler._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta) @@ -1306,22 +1324,21 @@ class AnthropicMessagesHandler(BaseTranslation): {index: chain((rewrite.arguments,), repeat("")) for index, rewrite in rewrites_by_block.items()} ) - def rewrite_tool_use(event: Mapping[str, object]) -> Mapping[str, object] | None: + def rewrite_tool_use(event: Mapping[str, object]) -> _SSEFieldRewrite | None: index: Final = event.get("index") if not isinstance(index, int) or index not in rewrites_by_block: return None match event.get("type"): case "content_block_start": - block: Final = event.get("content_block") name: Final = rewrites_by_block[index].name - if not isinstance(block, Mapping) or name is None: + if name is None: return None - return {**event, "content_block": {**block, "name": name}} + return _SSEFieldRewrite("content_block", "name", name) case "content_block_delta": delta: Final = event.get("delta") if not isinstance(delta, Mapping) or delta.get("type") != "input_json_delta": return None - return {**event, "delta": {**delta, "partial_json": next(argument_replacements[index])}} + return _SSEFieldRewrite("delta", "partial_json", next(argument_replacements[index])) case _: return None @@ -1343,8 +1360,7 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _rewrite_buffered_item(item: object, rewrite_event: _SSEEventRewriter) -> object: if isinstance(item, dict): - rewritten: Final = rewrite_event(_as_str_mapping(item)) - return item if rewritten is None else dict(rewritten) + return _rewritten_event(_as_str_mapping(item), rewrite_event) if isinstance(item, (bytes, bytearray)): return AnthropicMessagesHandler._rewrite_sse_events(bytes(item), rewrite_event) return item @@ -1374,8 +1390,8 @@ class AnthropicMessagesHandler(BaseTranslation): return line if not isinstance(data, dict): return line - rewritten: Final = rewrite_event(_as_str_mapping(data)) - return line if rewritten is None else "data: " + json.dumps(rewritten) + rewritten: Final = _rewritten_event(_as_str_mapping(data), rewrite_event) + return line if rewritten is data else "data: " + json.dumps(rewritten) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: stream_ended: Final = self._check_streaming_has_ended(responses_so_far) From 89e11949c834d0b466375b0442b27304946568a9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:22:30 -0700 Subject: [PATCH 118/411] fix(guardrails): key Responses stream tool-call rewrites by call_id Bridged Responses streams give reasoning and message items output_index 0 and start function calls at 1, so keying rewrites by output_index rewrote the wrong items. Rewrites now follow each function call's call_id through the buffered item and argument events, refuse when an event cannot be resolved to a rewritten call, and the refusal branches on all three handlers get regression tests --- .../guardrail_translation/handler.py | 132 ++++++++++++------ .../test_anthropic_guardrail_handler.py | 25 ++++ .../test_openai_guardrail_handler.py | 56 ++++++++ ...test_openai_responses_guardrail_handler.py | 90 ++++++++++++ 4 files changed, 259 insertions(+), 44 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 208ceb959e6..280a8670c36 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -140,6 +140,10 @@ _TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset( ) +_FUNCTION_CALL_ARGUMENT_EVENT_TYPES: Final = frozenset( + {"response.function_call_arguments.delta", "response.function_call_arguments.done"} +) +_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( {"function_call_output": "output", "message": "content"} ) @@ -930,70 +934,110 @@ class OpenAIResponsesHandler(BaseTranslation): ) -> None: """Write ended-stream guardrail tool-call rewrites into the completed envelope's ``function_call`` items and sync the earlier stream events, - keyed by ``output_index``. The guardrail sees the envelope's function - calls in output order, which is how a rewritten call finds its item; a - rewrite whose calls do not line up with the envelope is reported as - undeliverable, so the pipeline executor discards it and releases the - original events.""" + keyed by ``call_id``. The guardrail sees the envelope's function calls + in output order, which is how a rewritten call finds its ``call_id``; + the stream events find their call through the ``call_id`` on + ``output_item`` events and the ``item_id`` on argument events, since an + event's ``output_index`` need not match the envelope's (the chat bridge + numbers tool calls from 1 while the envelope lists them after the + message). A rewrite whose calls do not line up with the envelope, or + whose events cannot be found, is reported as undeliverable, so the + pipeline executor discards it and releases the original events.""" if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - function_call_indices: Final = tuple( - output_idx - for output_idx, output_item in enumerate(outputs) - if stream_item_field(output_item, "type") == "function_call" + function_call_items: Final = tuple( + output_item for output_item in outputs if stream_item_field(output_item, "type") == "function_call" ) - if len(function_call_indices) != len(post_guardrail_tool_calls): - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - - raise UndeliverableStreamRewrite(guardrail_name) - rewrites_by_output_index: Final = MappingProxyType( + call_ids: Final = tuple( + call_id + for output_item in function_call_items + if isinstance(call_id := stream_item_field(output_item, "call_id"), str) and call_id + ) + stream_events: Final = responses_so_far[:-1] + call_id_by_item_id: Final = self._function_call_ids_by_item_id(stream_events) + event_call_ids: Final = tuple( + self._function_call_event_call_id(event, call_id_by_item_id) for event in stream_events + ) + rewrites_by_call_id: Final = MappingProxyType( { - output_idx: after - for output_idx, before, after in zip( - function_call_indices, pre_guardrail_tool_calls, post_guardrail_tool_calls - ) + call_id: after + for call_id, before, after in zip(call_ids, pre_guardrail_tool_calls, post_guardrail_tool_calls) if after != before } ) - for output_idx, rewrite in rewrites_by_output_index.items(): - self._write_function_call_item(outputs[output_idx], rewrite.name, rewrite.arguments) - self._sync_stream_events_with_tool_call_rewrites( - stream_events=responses_so_far[:-1], - rewrites_by_output_index=rewrites_by_output_index, + unresolved_argument_event: Final = any( + call_id is None and stream_item_field(event, "type") in _FUNCTION_CALL_ARGUMENT_EVENT_TYPES + for event, call_id in zip(stream_events, event_call_ids) ) + if ( + len(call_ids) != len(function_call_items) + or len(frozenset(call_ids)) != len(call_ids) + or len(call_ids) != len(post_guardrail_tool_calls) + or unresolved_argument_event + or not rewrites_by_call_id.keys() <= frozenset(event_call_ids) + ): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - def _sync_stream_events_with_tool_call_rewrites( - self, - stream_events: Sequence[object], - rewrites_by_output_index: Mapping[int, _ToolCallShape], - ) -> None: - """Sync pre-completion function-call events with the rewritten completed - response: the first ``function_call_arguments.delta`` for a rewritten call - carries the full rewritten arguments and the rest are blanked, while - ``function_call_arguments.done`` and ``output_item.done`` carry the full - rewritten arguments and ``output_item.added`` / ``output_item.done`` the - rewritten name, so every event a client may read agrees with the - rewritten ``response.completed`` payload.""" + raise UndeliverableStreamRewrite(guardrail_name) + for output_item, rewrite in ( + (output_item, rewrites_by_call_id[call_id]) + for output_item, call_id in zip(function_call_items, call_ids) + if call_id in rewrites_by_call_id + ): + self._write_function_call_item(output_item, rewrite.name, rewrite.arguments) delta_replacements: Final = MappingProxyType( - {index: chain((rewrite.arguments,), repeat("")) for index, rewrite in rewrites_by_output_index.items()} + {call_id: chain((rewrite.arguments,), repeat("")) for call_id, rewrite in rewrites_by_call_id.items()} ) - for event in stream_events: - output_index = stream_item_field(event, "output_index") - if not isinstance(output_index, int) or output_index not in rewrites_by_output_index: + for event, call_id in zip(stream_events, event_call_ids): + if call_id not in rewrites_by_call_id: continue - rewrite = rewrites_by_output_index[output_index] match stream_item_field(event, "type"): case "response.function_call_arguments.delta": - self._write_event_field(event, "delta", next(delta_replacements[output_index])) + self._write_event_field(event, "delta", next(delta_replacements[call_id])) case "response.function_call_arguments.done": - self._write_event_field(event, "arguments", rewrite.arguments) + self._write_event_field(event, "arguments", rewrites_by_call_id[call_id].arguments) case "response.output_item.added": - self._write_function_call_item(stream_item_field(event, "item"), rewrite.name, None) + self._write_function_call_item( + stream_item_field(event, "item"), rewrites_by_call_id[call_id].name, None + ) case "response.output_item.done": - self._write_function_call_item(stream_item_field(event, "item"), rewrite.name, rewrite.arguments) + self._write_function_call_item( + stream_item_field(event, "item"), + rewrites_by_call_id[call_id].name, + rewrites_by_call_id[call_id].arguments, + ) case _: pass + @staticmethod + def _function_call_ids_by_item_id(stream_events: Sequence[object]) -> Mapping[str, str]: + items: Final = tuple( + stream_item_field(event, "item") + for event in stream_events + if stream_item_field(event, "type") in _OUTPUT_ITEM_EVENT_TYPES + ) + return MappingProxyType( + { + item_id: call_id + for item in items + if stream_item_field(item, "type") == "function_call" + and isinstance(item_id := stream_item_field(item, "id"), str) + and isinstance(call_id := stream_item_field(item, "call_id"), str) + } + ) + + @staticmethod + def _function_call_event_call_id(event: object, call_id_by_item_id: Mapping[str, str]) -> str | None: + event_type: Final = stream_item_field(event, "type") + if event_type in _FUNCTION_CALL_ARGUMENT_EVENT_TYPES: + item_id: Final = stream_item_field(event, "item_id") + return call_id_by_item_id.get(item_id) if isinstance(item_id, str) else None + if event_type not in _OUTPUT_ITEM_EVENT_TYPES: + return None + item: Final = stream_item_field(event, "item") + call_id: Final = stream_item_field(item, "call_id") + return call_id if stream_item_field(item, "type") == "function_call" and isinstance(call_id, str) else None + @staticmethod def _write_function_call_item(item: object, name: str | None, arguments: str | None) -> None: if item is None: diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index bd2dff9b33c..64c243343d5 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -381,6 +381,31 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: assert chunks == original + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_use_rewrite_with_server_tool_use_block_fails_closed(self): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = AnthropicMessagesHandler() + server_tool_use = [ + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"query": "fruit"}'}}), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ] + tool_use = self._ended_tool_use_sse_chunks() + chunks = ( + tool_use[:1] + + [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in server_tool_use] + + [chunk.replace(b'"index": 0', b'"index": 1') for chunk in tool_use[1:]] + ) + + with pytest.raises(UndeliverableStreamRewrite): + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + @pytest.mark.asyncio async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self): handler = AnthropicMessagesHandler() diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index be168ec83e6..c7011a6f00e 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1252,6 +1252,62 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: deliver_ended_stream_rewrites=True, ) + @staticmethod + def _two_choice_tool_call_stream_chunks() -> list: + from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponseStream, + StreamingChoices, + ) + + def chunk( + choice_index: int, tool_call: ChatCompletionDeltaToolCall | None, finish_reason: Optional[str] = None + ) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="gpt-4", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=choice_index, + delta=Delta(tool_calls=[tool_call] if tool_call else None), + finish_reason=finish_reason, + ) + ], + ) + + def fragment(arguments: str, name: Optional[str] = None, call_id: Optional[str] = None): + return ChatCompletionDeltaToolCall( + id=call_id, index=0, type="function", function=Function(name=name, arguments=arguments) + ) + + return [ + chunk(0, fragment("", name="lookup_fruit", call_id="call_1")), + chunk(1, fragment("", name="lookup_fruit", call_id="call_2")), + chunk(0, fragment('{"fruit": "persimmon"}')), + chunk(1, fragment('{"fruit": "durian"}')), + chunk(0, None, finish_reason="tool_calls"), + chunk(1, None, finish_reason="tool_calls"), + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = OpenAIChatCompletionsHandler() + chunks = self._two_choice_tool_call_stream_chunks() + + with pytest.raises(UndeliverableStreamRewrite): + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + @pytest.mark.asyncio async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self): handler = OpenAIChatCompletionsHandler() diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index bdf7f83757c..fa1e969b277 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -1314,6 +1314,96 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert completed_event.response.output[0].arguments == '{"fruit": "[MASKED]"}' assert completed_event.response.output[0].name == "lookup_fruit" + @staticmethod + def _bridged_function_call_stream_events() -> List[dict]: + reasoning = {"type": "reasoning", "id": "rs_1", "summary": []} + text = {"type": "output_text", "text": "Looking that up", "annotations": []} + message = {"type": "message", "id": "msg_1", "role": "assistant", "status": "completed", "content": [text]} + + def function_call(arguments: str, status: str) -> dict: + return { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "lookup_fruit", + "arguments": arguments, + "status": status, + } + + return [ + {"type": "response.output_item.added", "output_index": 0, "item": dict(reasoning)}, + {"type": "response.output_item.done", "output_index": 0, "item": dict(reasoning)}, + {"type": "response.output_item.added", "output_index": 0, "item": {**message, "status": "in_progress", "content": []}}, + {"type": "response.output_text.delta", "item_id": "msg_1", "output_index": 0, "content_index": 0, "delta": "Looking that up"}, + {"type": "response.output_item.done", "output_index": 0, "item": {**message, "content": [dict(text)]}}, + {"type": "response.output_item.added", "output_index": 1, "item": function_call("", "in_progress")}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_1", "output_index": 1, "delta": '{"fruit":'}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_1", "output_index": 1, "delta": ' "persimmon"}'}, + { + "type": "response.function_call_arguments.done", + "item_id": "fc_1", + "output_index": 1, + "arguments": '{"fruit": "persimmon"}', + }, + {"type": "response.output_item.done", "output_index": 1, "item": function_call('{"fruit": "persimmon"}', "completed")}, + { + "type": "response.completed", + "response": { + "id": "resp_1", + "model": "claude-haiku-4-5", + "output": [ + dict(reasoning), + {**message, "content": [dict(text)]}, + function_call('{"fruit": "persimmon"}', "completed"), + ], + }, + }, + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_keys_bridged_function_call_events_by_call_id(self): + handler = OpenAIResponsesHandler() + events = self._bridged_function_call_stream_events() + + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert events[6]["delta"] == '{"fruit": "[MASKED]"}' + assert events[7]["delta"] == "" + assert events[8]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[5]["item"]["name"] == "lookup_fruit" + assert events[9]["item"]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[10]["response"]["output"][2]["arguments"] == '{"fruit": "[MASKED]"}' + assert events[3]["delta"] == "Looking that up" + assert events[4]["item"]["content"][0]["text"] == "Looking that up" + assert events[10]["response"]["output"][1]["content"][0]["text"] == "Looking that up" + assert events[1]["item"] == {"type": "reasoning", "id": "rs_1", "summary": []} + + @pytest.mark.asyncio + @pytest.mark.parametrize("mismatch", ["orphan_call_id", "duplicate_call_id"]) + async def test_deliver_ended_stream_function_call_rewrite_without_matching_events_fails_closed(self, mismatch): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = OpenAIResponsesHandler() + events = self._ended_function_call_stream_events() + envelope_item = events[5]["response"]["output"][0] + if mismatch == "orphan_call_id": + events[5]["response"]["output"] = [{**envelope_item, "call_id": "call_999"}] + else: + events[5]["response"]["output"] = [dict(envelope_item), dict(envelope_item)] + + with pytest.raises(UndeliverableStreamRewrite): + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + @pytest.mark.asyncio async def test_ended_stream_function_call_rewrite_leaves_events_untouched_by_default(self): handler = OpenAIResponsesHandler() From 071f83980c319b9d9a21f77179781052c050b517 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:26:17 -0700 Subject: [PATCH 119/411] refactor(proxy): type the post_call deferral helpers' request-state parameters --- litellm/proxy/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index e48bd3851fb..e653ca6b62c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -544,7 +544,7 @@ def _guardrails_outside_pipeline(policy_name: str, pipeline: "GuardrailPipeline" def _without_names( - bucket: dict, # mutable-ok: the applied_* header slots live in the request-state dict every hook writes in place + bucket: dict[str, object], # mutable-ok: the applied_* header slots live in the request-state dict hooks write slot: str, names: frozenset[str], ) -> None: @@ -561,7 +561,7 @@ def _without_names( def _withdraw_deferred_claims( - data: dict, # mutable-ok: same request-payload shape as post_call_success_hook's data + data: dict[str, object], # mutable-ok: same request-payload shape as post_call_success_hook's data deferred: Sequence[tuple[str, "GuardrailPipeline"]], ) -> None: outside_by_policy: Final = MappingProxyType( @@ -586,7 +586,7 @@ def _withdraw_deferred_claims( def _defer_post_call_pipelines( - data: dict, # mutable-ok: same request-payload shape as post_call_success_hook's data + data: dict[str, object], # mutable-ok: same request-payload shape as post_call_success_hook's data response: ResponsesAPIResponse, ) -> None: deferred: Final = _post_call_pipelines(data) @@ -2990,7 +2990,7 @@ class ProxyLogging: async def _run_post_call_pipelines( self, - data: dict, # mutable-ok: same request-payload shape as post_call_success_hook's data + data: dict[str, object], # mutable-ok: same request-payload shape as post_call_success_hook's data user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes, ) -> LLMResponseTypes | None: From 0c2d0f4777f4113b42510e2daef2330ca0bb87a8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:28:13 -0700 Subject: [PATCH 120/411] fix(spend-tracking): forget an empty spend-log lookup after thirty seconds The daily spend rows and the spend logs of one batch are written a moment apart, so a usage read landing between them used to remember the session as nameless for ten minutes on that worker. Found identities keep the ten minute entry --- litellm/constants.py | 1 + .../spend_tracking/key_metadata_recovery.py | 19 ++++++++++++++++-- .../test_key_metadata_recovery.py | 20 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 9654c84c412..cfc5b6b86a7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1764,6 +1764,7 @@ SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)) SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600 +SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30 SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000 # Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated # callers from forcing a DB query per request for unknown names, while bounding diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 7212d8b6acb..354c6479d34 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -9,7 +9,11 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache -from litellm.constants import SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS, SPEND_LOG_KEY_METADATA_CACHE_TTL +from litellm.constants import ( + SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS, + SPEND_LOG_KEY_METADATA_CACHE_TTL, + SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL, +) from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash from litellm.proxy.utils import PrismaClient from litellm.repositories.user_repository import UserRepository @@ -230,6 +234,17 @@ async def _query_spend_log_metadata( ) +def _remember_spend_log_metadata( + cache: InMemoryCache, digest: str, window: tuple[datetime, datetime], meta: KeyMetadataDict | None +) -> None: + if meta is None: + cache.set_cache( + _spend_log_cache_key(digest, window), KeyMetadataDict(), ttl=SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL + ) + return + cache.set_cache(_spend_log_cache_key(digest, window), meta) + + async def recover_key_metadata_from_spend_logs( prisma_client: PrismaClient, missing_keys: AbstractSet[str], @@ -251,7 +266,7 @@ async def recover_key_metadata_from_spend_logs( fresh: Final = await _query_spend_log_metadata(prisma_client, uncached, window) if uncached else _EMPTY_KEY_METADATA if fresh is not None: for digest in uncached: - cache.set_cache(_spend_log_cache_key(digest, window), fresh.get(digest, KeyMetadataDict())) + _remember_spend_log_metadata(cache, digest, window, fresh.get(digest)) return MappingProxyType( {digest: meta for digest, meta in (*cached.items(), *(fresh or _EMPTY_KEY_METADATA).items()) if meta} ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 3d7362b9de0..4f065070330 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -1,3 +1,4 @@ +import time from collections.abc import Sequence from datetime import datetime from types import SimpleNamespace @@ -7,6 +8,7 @@ import pytest from prisma.errors import PrismaError from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import SPEND_LOG_KEY_METADATA_CACHE_TTL, SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL from litellm.proxy.spend_tracking.key_metadata_recovery import ( fill_missing_api_key_aliases, recover_double_hashed_key_metadata, @@ -395,3 +397,21 @@ async def test_recover_key_metadata_from_spend_logs_does_not_cache_a_failed_quer result = await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) assert result[digest]["key_alias"] == "back-online" + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_forgets_a_miss_long_before_a_hit(): + found = hash_token("cli-session-found") + unknown = hash_token("cli-session-unknown") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + cache = InMemoryCache(default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(found, "found-alias", None, None)]) + started = time.time() + + await recover_key_metadata_from_spend_logs(mock_prisma, {found, unknown}, window, cache=cache) + + hit_expires = next(deadline for key, deadline in cache.ttl_dict.items() if found in key) + miss_expires = next(deadline for key, deadline in cache.ttl_dict.items() if unknown in key) + assert miss_expires - started <= SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL + 1 + assert hit_expires - started >= SPEND_LOG_KEY_METADATA_CACHE_TTL - 1 From c1132d045470853a8a186d730678493f3f015dbe Mon Sep 17 00:00:00 2001 From: shotsan Date: Tue, 8 Sep 2026 13:44:50 -0700 Subject: [PATCH 121/411] fix(convert_dict_to_response): handle empty choices list without raising 500 APIError (Fixes #40276) --- .../convert_dict_to_response.py | 6 +- .../test_convert_dict_to_response.py | 70 ++++++++++++++++++- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index a6f10e1ede3..7d8072ba622 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -179,7 +179,7 @@ async def convert_to_streaming_response_async( choice_list: Final[list[StreamingChoices]] = [] - if not response_object.get("choices"): + if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): from litellm.exceptions import APIError raise APIError( @@ -287,7 +287,7 @@ def convert_to_streaming_response( model_response_object: Final = ModelResponseStream() choice_list: Final[list[StreamingChoices]] = [] - if not response_object.get("choices"): + if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): from litellm.exceptions import APIError raise APIError( @@ -623,7 +623,7 @@ def convert_to_model_response_object( return convert_to_streaming_response(response_object=response_object) choice_list: Final[list[Choices]] = [] - if not response_object.get("choices") or not isinstance(response_object["choices"], Iterable): + if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): from litellm.exceptions import APIError raise APIError( diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 304d732c518..5406df691aa 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1,4 +1,4 @@ - +import pytest from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -99,3 +99,71 @@ def test_handle_invalid_parallel_tool_calls_skips_custom_tool_calls(): ) result = _handle_invalid_parallel_tool_calls([custom_tool_call, function_tool_call]) assert result == [custom_tool_call, function_tool_call] + + +def test_convert_empty_choices_response() -> None: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response, + ) + + resp = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, + "vertex_ai_safety_results": ["blocked"], + } + result = convert_to_model_response_object( + response_object=resp, + model_response_object=ModelResponse(), + response_type="completion", + ) + assert result.choices == [] + assert getattr(result, "vertex_ai_safety_results") == ["blocked"] + + # Test sync streaming generator handles empty choices + sync_stream = list(convert_to_streaming_response(response_object=resp)) + assert len(sync_stream) == 1 + assert sync_stream[0].choices == [] + + +@pytest.mark.asyncio +async def test_convert_empty_choices_response_async() -> None: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response_async, + ) + + resp = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, + } + async_chunks = [] + async for chunk in convert_to_streaming_response_async(response_object=resp): + async_chunks.append(chunk) + assert len(async_chunks) == 1 + assert async_chunks[0].choices == [] + + +def test_convert_missing_choices_raises_api_error() -> None: + from litellm.exceptions import APIError + + resp = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + } + with pytest.raises(APIError) as exc_info: + convert_to_model_response_object( + response_object=resp, + model_response_object=ModelResponse(), + response_type="completion", + ) + assert "no 'choices'" in str(exc_info.value) + From d72eb4491ab2da2a56944c720a3200280cdd040d Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 8 Sep 2026 17:22:30 -0400 Subject: [PATCH 122/411] fix(ui): repair pass-through delete confirm dialog and disable delete for config endpoints --- .../PassThroughEndpointsTable.test.tsx | 47 ++++++++++++++ .../PassThroughEndpointsTableColumns.tsx | 30 +++++++-- .../PassThroughSettings.tsx | 62 ++++++++----------- 3 files changed, 98 insertions(+), 41 deletions(-) diff --git a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTable.test.tsx b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTable.test.tsx index 040fa463f9c..7a3fc621fc1 100644 --- a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTable.test.tsx @@ -89,6 +89,53 @@ describe("PassThroughEndpointsTable", () => { expect(onDeleteClick).toHaveBeenCalledWith("ep-1"); }); + it("should disable edit and delete for config-defined endpoints", async () => { + const user = userEvent.setup(); + const onEndpointClick = vi.fn(); + const onDeleteClick = vi.fn(); + const configEndpoint: passThroughItem = { + id: "ep-config", + path: "/from-config", + target: "https://config.example.com", + headers: {}, + is_from_config: true, + }; + render( + , + ); + + await user.click(screen.getByTestId("endpoint-actions-ep-config")); + const editItem = await screen.findByTestId("endpoint-action-edit"); + const deleteItem = await screen.findByTestId("endpoint-action-delete"); + + expect(editItem).toHaveAttribute("data-disabled"); + expect(deleteItem).toHaveAttribute("data-disabled"); + + await user.click(editItem); + await user.click(deleteItem); + + expect(onEndpointClick).not.toHaveBeenCalled(); + expect(onDeleteClick).not.toHaveBeenCalled(); + }); + + it("should label endpoint source as Config or DB", () => { + const configEndpoint: passThroughItem = { + id: "ep-config", + path: "/from-config", + target: "https://config.example.com", + headers: {}, + is_from_config: true, + }; + render(); + expect(screen.getByText("Config")).toBeInTheDocument(); + expect(screen.getAllByText("DB")).toHaveLength(2); + }); + it("should disable edit and delete for endpoints without an id", async () => { const user = userEvent.setup(); const onEndpointClick = vi.fn(); diff --git a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTableColumns.tsx b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTableColumns.tsx index d22b274861a..d63be628110 100644 --- a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTableColumns.tsx @@ -18,6 +18,10 @@ import { cn } from "@/lib/cva.config"; import type { passThroughItem } from "./PassThroughSettings"; +const CONFIG_EDIT_HINT = "Config pass-through endpoints cannot be edited on the dashboard. Please edit the config file."; +const CONFIG_DELETE_HINT = + "Config pass-through endpoints cannot be deleted on the dashboard. Please edit the config file."; + function HeaderWithTooltip({ title, tooltip }: { title: string; tooltip: string }) { return (
@@ -73,6 +77,7 @@ interface EndpointRowActionsProps { function EndpointRowActions({ endpoint, onEndpointClick, onDeleteClick }: EndpointRowActionsProps) { const endpointId = endpoint.id; + const isFromConfig = endpoint.is_from_config ?? false; return ( endpointId && onEndpointClick(endpointId)} + disabled={isFromConfig || !endpointId} + title={isFromConfig ? CONFIG_EDIT_HINT : undefined} + onClick={() => !isFromConfig && endpointId && onEndpointClick(endpointId)} > Edit @@ -95,8 +101,9 @@ function EndpointRowActions({ endpoint, onEndpointClick, onDeleteClick }: Endpoi endpointId && onDeleteClick(endpointId)} + disabled={isFromConfig || !endpointId} + title={isFromConfig ? CONFIG_DELETE_HINT : undefined} + onClick={() => !isFromConfig && endpointId && onDeleteClick(endpointId)} > Delete @@ -124,7 +131,9 @@ export const getPassThroughEndpointsTableColumns = ({ enableSorting: false, cell: ({ row }) => { const endpointId = row.original.id; - if (!endpointId) return —; + if (!endpointId || row.original.is_from_config) { + return —; + } return ( { + const isFromConfig = row.original.is_from_config ?? false; + return ; + }, + }, { id: "path", accessorKey: "path", diff --git a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughSettings.tsx b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughSettings.tsx index 2dc6fdbd32c..8ef2766d412 100644 --- a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughSettings.tsx +++ b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughSettings.tsx @@ -1,4 +1,13 @@ import React, { useState, useEffect } from "react"; +import { + AlertDialog, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; import { Button } from "@/components/ui/button"; import { deletePassThroughEndpointsCall, getPassThroughEndpointsCall } from "../networking"; import AddPassThroughEndpoint from "../add_pass_through"; @@ -25,6 +34,7 @@ export interface passThroughItem { methods?: string[]; guardrails?: Record; default_query_params?: Record; + is_from_config?: boolean; } const PassThroughSettings: React.FC = ({ accessToken, userRole, userID, premiumUser }) => { @@ -133,42 +143,22 @@ const PassThroughSettings: React.FC = ({ accessToken, onDeleteClick={handleDelete} /> - {isDeleteModalOpen && ( -
-
- - - - -
-
-
-
-

Delete Pass-Through Endpoint

-
-

- Are you sure you want to delete this pass-through endpoint? This action cannot be undone. -

-
-
-
-
-
- - -
-
-
-
- )} + !open && cancelDelete()}> + + + Delete Pass-Through Endpoint + + Are you sure you want to delete this pass-through endpoint? This action cannot be undone. + + + + Cancel + + + +
); }; From 093b473710313f22efd864e5262e5a376a0ee2ce Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 8 Sep 2026 17:43:23 -0400 Subject: [PATCH 123/411] fix(ui): surface the config-endpoint hint as visible menu text --- .../PassThroughEndpointsTable.test.tsx | 12 ++++++++++++ .../PassThroughEndpointsTableColumns.tsx | 12 +++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTable.test.tsx b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTable.test.tsx index 7a3fc621fc1..af3f98b746b 100644 --- a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTable.test.tsx @@ -115,6 +115,9 @@ describe("PassThroughEndpointsTable", () => { expect(editItem).toHaveAttribute("data-disabled"); expect(deleteItem).toHaveAttribute("data-disabled"); + expect(screen.getByTestId("endpoint-config-hint")).toHaveTextContent( + "This endpoint is defined in the config file and cannot be edited or deleted on the dashboard.", + ); await user.click(editItem); await user.click(deleteItem); @@ -123,6 +126,15 @@ describe("PassThroughEndpointsTable", () => { expect(onDeleteClick).not.toHaveBeenCalled(); }); + it("should not show the config hint for DB endpoints", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByTestId("endpoint-actions-ep-1")); + await screen.findByTestId("endpoint-action-delete"); + expect(screen.queryByTestId("endpoint-config-hint")).not.toBeInTheDocument(); + }); + it("should label endpoint source as Config or DB", () => { const configEndpoint: passThroughItem = { id: "ep-config", diff --git a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTableColumns.tsx b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTableColumns.tsx index d63be628110..5b18685a140 100644 --- a/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PassThroughSettings/PassThroughEndpointsTableColumns.tsx @@ -18,9 +18,8 @@ import { cn } from "@/lib/cva.config"; import type { passThroughItem } from "./PassThroughSettings"; -const CONFIG_EDIT_HINT = "Config pass-through endpoints cannot be edited on the dashboard. Please edit the config file."; -const CONFIG_DELETE_HINT = - "Config pass-through endpoints cannot be deleted on the dashboard. Please edit the config file."; +const CONFIG_ENDPOINT_HINT = + "This endpoint is defined in the config file and cannot be edited or deleted on the dashboard."; function HeaderWithTooltip({ title, tooltip }: { title: string; tooltip: string }) { return ( @@ -91,7 +90,6 @@ function EndpointRowActions({ endpoint, onEndpointClick, onDeleteClick }: Endpoi !isFromConfig && endpointId && onEndpointClick(endpointId)} > @@ -102,12 +100,16 @@ function EndpointRowActions({ endpoint, onEndpointClick, onDeleteClick }: Endpoi variant="destructive" data-testid="endpoint-action-delete" disabled={isFromConfig || !endpointId} - title={isFromConfig ? CONFIG_DELETE_HINT : undefined} onClick={() => !isFromConfig && endpointId && onDeleteClick(endpointId)} > Delete + {isFromConfig && ( +
+ {CONFIG_ENDPOINT_HINT} +
+ )} ); From 547f81c1a52722f487597d93975df1fa79e9464e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:46:23 -0700 Subject: [PATCH 124/411] test: type the background response retrieval test helpers --- .../proxy/policy_engine/test_response_retrieval.py | 9 ++++++--- .../test_litellm/proxy/test_common_request_processing.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py index 8c00b18eb63..b704f666646 100644 --- a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py +++ b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py @@ -1,4 +1,5 @@ import logging +from collections.abc import Mapping import pytest @@ -76,14 +77,16 @@ def policy_engine(): attachment_registry.clear() -def _retrieval_data(model_id: str) -> dict: +def _retrieval_data(model_id: str) -> dict[str, object]: return {"response_id": _encoded_response_id(model_id), "litellm_metadata": {}} -def _attached_pipelines(data: dict) -> tuple[tuple[str, str], ...]: +def _attached_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, str], ...]: + bucket = data["litellm_metadata"] + assert isinstance(bucket, dict) return tuple( (policy_name, ",".join(step.guardrail for step in pipeline.steps)) - for policy_name, pipeline in data["litellm_metadata"]["_guardrail_pipelines"] + for policy_name, pipeline in bucket["_guardrail_pipelines"] ) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 08007e22cfb..c451a3b4cb0 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -8340,7 +8340,7 @@ class TestBackgroundResponseRetrievalGovernance: ) return router - async def _pre_call(self, route_type: str, monkeypatch) -> dict: + async def _pre_call(self, route_type: str, monkeypatch: pytest.MonkeyPatch) -> dict[str, object]: from litellm.responses.utils import ResponsesAPIRequestUtils client_facing_response_id = "resp_opaque-client-facing-id" From e700e79cce9dac5d0fda0b5d31210283230c9de7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:52:38 -0700 Subject: [PATCH 125/411] refactor(guardrails): keep the rescanned inputs instead of rebuilding the texts list --- litellm/proxy/policy_engine/pipeline_executor.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index afc9a29afd9..089f0683823 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -128,7 +128,7 @@ class _StreamRewriteObserver(CustomGuardrail): class _ScannedTextRecorder(CustomGuardrail): def __init__(self, guardrail_name: str) -> None: super().__init__(guardrail_name=guardrail_name) - self.texts: tuple[str, ...] | None = None + self.inputs: GenericGuardrailAPIInputs | None = None @_logged_by_inner_guardrail async def apply_guardrail( @@ -138,7 +138,7 @@ class _ScannedTextRecorder(CustomGuardrail): input_type: Literal["request", "response"], logging_obj: "LiteLLMLoggingObj | None" = None, ) -> GenericGuardrailAPIInputs: - self.texts = _text_snapshot(inputs.get("texts")) + self.inputs = inputs return inputs @@ -183,12 +183,16 @@ class _LegacyHookStreamAdapter(CustomGuardrail): if replacement is None: return inputs scanned: Final = _text_snapshot(inputs.get("texts")) - rewritten: Final = await self._scanned_texts(replacement, logging_obj) + rescanned: Final = await self._rescan(replacement, logging_obj) + rewritten: Final = None if rescanned is None else rescanned.get("texts") if scanned is None or rewritten is None or len(rewritten) != len(scanned): raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") - return {**inputs, "texts": list(rewritten)} + rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten} + return rewritten_inputs - async def _scanned_texts(self, response: object, logging_obj: "LiteLLMLoggingObj | None") -> tuple[str, ...] | None: + async def _rescan( + self, response: object, logging_obj: "LiteLLMLoggingObj | None" + ) -> GenericGuardrailAPIInputs | None: recorder: Final = _ScannedTextRecorder(self.guardrail_name or "unknown") await self.endpoint_translation.process_output_response( response=response, @@ -196,7 +200,7 @@ class _LegacyHookStreamAdapter(CustomGuardrail): litellm_logging_obj=logging_obj, user_api_key_dict=self.user_api_key_dict, ) - return recorder.texts + return recorder.inputs def _prepare_hook_input( From 907c200b3162368cfaa0e363411997330b2da4f2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:00:56 -0700 Subject: [PATCH 126/411] fix(spend-tracking): run one spend-log scan at a time and back off repeated misses Concurrent usage reads on one worker now share a single spend-log query instead of each scanning the same window, and a digest that comes back nameless a second time is remembered for the full ten minutes rather than thirty seconds, so a key that never resolves costs at most two scans per worker per window per ten minutes. The first miss still expires after thirty seconds so a read that lands between the daily spend flush and the spend-log flush recovers on the next read --- .../spend_tracking/key_metadata_recovery.py | 71 +++++++++++++------ .../test_key_metadata_recovery.py | 44 ++++++++++++ 2 files changed, 92 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 354c6479d34..319050b87cd 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -1,3 +1,4 @@ +import asyncio from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet from datetime import datetime @@ -77,6 +78,7 @@ _SPEND_LOG_METADATA_CACHE: Final = InMemoryCache( max_size_in_memory=SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS, default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL, ) +_SPEND_LOG_QUERY_LOCK: Final = asyncio.Lock() _EMPTY_KEY_METADATA: Final[Mapping[str, KeyMetadataDict]] = MappingProxyType({}) _EMPTY_EMAILS: Final[Mapping[str, str]] = MappingProxyType({}) @@ -205,11 +207,17 @@ def _spend_log_cache_key(digest: str, window: tuple[datetime, datetime]) -> str: def _cached_spend_log_metadata( cache: InMemoryCache, - digest: str, + digests: AbstractSet[str], window: tuple[datetime, datetime], -) -> KeyMetadataDict | None: - cached: Final[object] = cache.get_cache(_spend_log_cache_key(digest, window)) - return None if cached is None else _CACHED_KEY_METADATA.validate_python(cached) +) -> Mapping[str, KeyMetadataDict]: + return MappingProxyType( + { + digest: _CACHED_KEY_METADATA.validate_python(cached) + for digest in digests + for cached in (cache.get_cache(_spend_log_cache_key(digest, window)),) + if cached is not None + } + ) async def _query_spend_log_metadata( @@ -237,12 +245,36 @@ async def _query_spend_log_metadata( def _remember_spend_log_metadata( cache: InMemoryCache, digest: str, window: tuple[datetime, datetime], meta: KeyMetadataDict | None ) -> None: - if meta is None: - cache.set_cache( - _spend_log_cache_key(digest, window), KeyMetadataDict(), ttl=SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL - ) + key: Final = _spend_log_cache_key(digest, window) + if meta is not None: + cache.set_cache(key, meta) return - cache.set_cache(_spend_log_cache_key(digest, window), meta) + missed_before: Final = f"{key}:missed-before" + if cache.get_cache(missed_before) is not None: + cache.set_cache(key, KeyMetadataDict()) + return + cache.set_cache(key, KeyMetadataDict(), ttl=SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL) + cache.set_cache(missed_before, True) + + +async def _spend_log_metadata_one_query_at_a_time( + prisma_client: PrismaClient, + cache: InMemoryCache, + lock: asyncio.Lock, + digests: AbstractSet[str], + window: tuple[datetime, datetime], +) -> Mapping[str, KeyMetadataDict]: + async with lock: + settled: Final = _cached_spend_log_metadata(cache, digests, window) + pending: Final = digests - frozenset(settled) + fresh: Final = ( + await _query_spend_log_metadata(prisma_client, pending, window) if pending else _EMPTY_KEY_METADATA + ) + if fresh is None: + return settled + for digest in pending: + _remember_spend_log_metadata(cache, digest, window, fresh.get(digest)) + return MappingProxyType({**settled, **fresh}) async def recover_key_metadata_from_spend_logs( @@ -250,26 +282,19 @@ async def recover_key_metadata_from_spend_logs( missing_keys: AbstractSet[str], window: tuple[datetime, datetime], cache: InMemoryCache = _SPEND_LOG_METADATA_CACHE, + lock: asyncio.Lock = _SPEND_LOG_QUERY_LOCK, ) -> Mapping[str, KeyMetadataDict]: digests: Final = frozenset(key for key in missing_keys if _is_spend_log_digest(key)) if not digests: return _EMPTY_KEY_METADATA - cached: Final = MappingProxyType( - { - digest: meta - for digest in digests - for meta in (_cached_spend_log_metadata(cache, digest, window),) - if meta is not None - } - ) + cached: Final = _cached_spend_log_metadata(cache, digests, window) uncached: Final = digests - frozenset(cached) - fresh: Final = await _query_spend_log_metadata(prisma_client, uncached, window) if uncached else _EMPTY_KEY_METADATA - if fresh is not None: - for digest in uncached: - _remember_spend_log_metadata(cache, digest, window, fresh.get(digest)) - return MappingProxyType( - {digest: meta for digest, meta in (*cached.items(), *(fresh or _EMPTY_KEY_METADATA).items()) if meta} + settled: Final = ( + await _spend_log_metadata_one_query_at_a_time(prisma_client, cache, lock, uncached, window) + if uncached + else _EMPTY_KEY_METADATA ) + return MappingProxyType({digest: meta for digest, meta in (*cached.items(), *settled.items()) if meta}) def _row_with_recovered_fields( diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 4f065070330..5b45777502a 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -1,3 +1,4 @@ +import asyncio import time from collections.abc import Sequence from datetime import datetime @@ -415,3 +416,46 @@ async def test_recover_key_metadata_from_spend_logs_forgets_a_miss_long_before_a miss_expires = next(deadline for key, deadline in cache.ttl_dict.items() if unknown in key) assert miss_expires - started <= SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL + 1 assert hit_expires - started >= SPEND_LOG_KEY_METADATA_CACHE_TTL - 1 + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_runs_one_query_for_concurrent_lookups(): + digest = hash_token("cli-session-shared") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + cache = InMemoryCache() + lock = asyncio.Lock() + mock_prisma = MagicMock() + + async def slow_query_raw(sql: str, *params: object) -> list[dict[str, str | None]]: + await asyncio.sleep(0.01) + return [_digest_row(digest, "shared-alias", None, None)] + + mock_prisma.db.query_raw = AsyncMock(side_effect=slow_query_raw) + + results = await asyncio.gather( + *( + recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache, lock=lock) + for _ in range(9) + ) + ) + + assert all(result[digest]["key_alias"] == "shared-alias" for result in results) + assert mock_prisma.db.query_raw.await_count == 1 + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_keeps_a_repeated_miss_as_long_as_a_hit(): + unknown = hash_token("cli-session-never-named") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + cache = InMemoryCache(default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs([]) + await recover_key_metadata_from_spend_logs(mock_prisma, {unknown}, window, cache=cache) + first_miss_key = next(key for key in cache.ttl_dict if unknown in key and not key.endswith(":missed-before")) + cache.ttl_dict[first_miss_key] = time.time() - 1 + started = time.time() + + await recover_key_metadata_from_spend_logs(mock_prisma, {unknown}, window, cache=cache) + + assert mock_prisma.db.query_raw.await_count == 2 + assert cache.ttl_dict[first_miss_key] - started >= SPEND_LOG_KEY_METADATA_CACHE_TTL - 1 From 345298f3c90871a7099952cb08e1e19d396883ee Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:04:19 -0700 Subject: [PATCH 127/411] fix(policy_engine): warn when a poll cannot re-match the submitted model name and keep default_on pre_call claims --- .../proxy/policy_engine/response_retrieval.py | 21 +++++++ litellm/proxy/utils.py | 14 ++++- .../policy_engine/test_response_retrieval.py | 56 ++++++++++++++++++- .../proxy_logging/test_guardrail_pipeline.py | 24 ++++++++ 4 files changed, 111 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/policy_engine/response_retrieval.py b/litellm/proxy/policy_engine/response_retrieval.py index 8fc75e40070..d284c44397e 100644 --- a/litellm/proxy/policy_engine/response_retrieval.py +++ b/litellm/proxy/policy_engine/response_retrieval.py @@ -18,6 +18,7 @@ from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_resolver import PolicyResolver from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.types.proxy.policy_engine import PolicyMatchContext from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline @@ -46,9 +47,29 @@ def _model_group_for_response_id(response_id: object, llm_router: "Router | None deployment: Final = llm_router.get_deployment(model_id) if deployment is None: return UngovernedRetrieval("deployment no longer in the router") + hidden_by: Final = _submit_model_hidden_by(deployment.model_name, llm_router.model_group_alias) + if hidden_by is not None: + verbose_proxy_logger.warning( + "Policy engine: background response %s re-matches policies on retrieval as model group %s (%s), " + "so a policy attached to the model name it was submitted as does not run on it", + response_id, + deployment.model_name, + hidden_by, + ) return deployment.model_name +def _submit_model_hidden_by(model_group: str, model_group_alias: Mapping[str, object]) -> str | None: + if "*" in model_group: + return "a wildcard deployment" + aliases: Final = tuple( + alias for alias in model_group_alias if resolve_model_group_alias(model_group_alias, alias) == model_group + ) + if not aliases: + return None + return f"the target of model_group_alias {', '.join(aliases)}" + + def _retrieval_context( data: Mapping[str, object], user_api_key_dict: "UserAPIKeyAuth", model_group: str ) -> PolicyMatchContext: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index e653ca6b62c..37ca52d6fc9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -543,6 +543,16 @@ def _guardrails_outside_pipeline(policy_name: str, pipeline: "GuardrailPipeline" return frozenset(resolved.guardrails) - frozenset(step.guardrail for step in pipeline.steps) +def _guardrails_run_standalone_pre_call(data: Mapping[str, object]) -> frozenset[str]: + return frozenset( + callback.guardrail_name + for callback in litellm.callbacks + if isinstance(callback, CustomGuardrail) + and callback.guardrail_name is not None + and callback.should_run_guardrail(data=data, event_type=GuardrailEventHooks.pre_call) + ) + + def _without_names( bucket: dict[str, object], # mutable-ok: the applied_* header slots live in the request-state dict hooks write slot: str, @@ -567,7 +577,9 @@ def _withdraw_deferred_claims( outside_by_policy: Final = MappingProxyType( {policy_name: _guardrails_outside_pipeline(policy_name, pipeline) for policy_name, pipeline in deferred} ) - running_elsewhere: Final = _pipeline_managed_guardrail_names(data, "pre_call").union(*outside_by_policy.values()) + running_elsewhere: Final = _pipeline_managed_guardrail_names(data, "pre_call").union( + _guardrails_run_standalone_pre_call(data), *outside_by_policy.values() + ) withdrawn_policies: Final = frozenset(name for name, outside in outside_by_policy.items() if not outside) withdrawn_guardrails: Final = _pipeline_step_guardrail_names(deferred) - running_elsewhere _, bucket = get_or_create_metadata_bucket(data) diff --git a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py index b704f666646..9dcbef888ce 100644 --- a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py +++ b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py @@ -14,11 +14,14 @@ GOVERNED_MODEL_GROUP = "gpt-5.4-mini" GOVERNED_MODEL_ID = "deployment-governed" UNGOVERNED_MODEL_GROUP = "gpt-4.1-mini" UNGOVERNED_MODEL_ID = "deployment-ungoverned" +WILDCARD_MODEL_GROUP = "openai/*" +WILDCARD_MODEL_ID = "deployment-wildcard" class FakeRouter: - def __init__(self, deployments: dict[str, Deployment]): + def __init__(self, deployments: dict[str, Deployment], model_group_alias: dict[str, object] | None = None): self._deployments = deployments + self.model_group_alias = model_group_alias or {} def get_deployment(self, model_id: str) -> Deployment | None: return self._deployments.get(model_id) @@ -32,12 +35,14 @@ def _deployment(model_group: str, model_id: str) -> Deployment: ) -def _router() -> FakeRouter: +def _router(model_group_alias: dict[str, object] | None = None) -> FakeRouter: return FakeRouter( { GOVERNED_MODEL_ID: _deployment(GOVERNED_MODEL_GROUP, GOVERNED_MODEL_ID), UNGOVERNED_MODEL_ID: _deployment(UNGOVERNED_MODEL_GROUP, UNGOVERNED_MODEL_ID), - } + WILDCARD_MODEL_ID: _deployment(WILDCARD_MODEL_GROUP, WILDCARD_MODEL_ID), + }, + model_group_alias, ) @@ -133,6 +138,51 @@ def test_already_attached_policy_is_not_attached_twice(policy_engine): assert data["litellm_metadata"]["applied_policies"] == ["response-governance"] +def _hidden_submit_model_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and "the model name it was submitted as" in record.getMessage() + ] + + +def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns(policy_engine, caplog): + data = _retrieval_data(WILDCARD_MODEL_ID) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + + assert data == _retrieval_data(WILDCARD_MODEL_ID) + assert [ + "as model group openai/* (a wildcard deployment)" in message for message in _hidden_submit_model_warnings(caplog) + ] == [True] + + +def test_aliased_model_group_still_attaches_its_own_policies_and_warns(policy_engine, caplog): + data = _retrieval_data(GOVERNED_MODEL_ID) + router = _router({"gpt-mini": GOVERNED_MODEL_GROUP, "gpt-hidden": {"model": GOVERNED_MODEL_GROUP, "hidden": True}}) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) + + assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) + assert [ + "(the target of model_group_alias gpt-mini, gpt-hidden)" in message + for message in _hidden_submit_model_warnings(caplog) + ] == [True] + + +def test_plain_model_group_retrieval_does_not_warn_about_the_submitted_model(policy_engine, caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + attach_post_call_pipelines_to_retrieval( + data=_retrieval_data(GOVERNED_MODEL_ID), + user_api_key_dict=UserAPIKeyAuth(), + llm_router=_router({"other-alias": UNGOVERNED_MODEL_GROUP}), + ) + + assert _hidden_submit_model_warnings(caplog) == [] + + def _ungoverned_retrieval_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: return [ record.getMessage() diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 8da07f39fd8..7e0736d4e67 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -1561,6 +1561,30 @@ async def test_pending_background_response_keeps_the_claim_of_a_policy_that_runs assert data["metadata"]["policy_sources"] == {"input-and-output-governance": "model:m"} +@pytest.mark.asyncio +async def test_pending_background_response_keeps_the_claim_of_a_default_on_guardrail_that_ran_pre_call( + proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry +): + class DualStageGuardrail(CustomGuardrail): + async def async_post_call_success_hook(self, data, user_api_key_dict, response): + return response + + monkeypatch.setattr( + litellm, + "callbacks", + [DualStageGuardrail(guardrail_name="gr-post", event_hook=["pre_call", "post_call"], default_on=True)], + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _claimed_post_call_pipeline_data("response-governance") + + await proxy_logging.post_call_success_hook( + data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth() + ) + + assert "applied_policies" not in data["metadata"] + assert data["metadata"]["applied_guardrails"] == ["gr-post"] + + @pytest.mark.asyncio async def test_retrieved_background_response_keeps_the_policy_claim_once_its_pipeline_ran( proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry From 00381ef03baedc9c9f7bec55f8c6139ecdb75f21 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 15:07:18 -0700 Subject: [PATCH 128/411] test: validate opaque stream IDs and hide log-reader credentials --- .../test_responses_bridge_streaming_e2e.py | 24 ++++++++++++------- tests/e2e/logging/datadog_reader.py | 10 ++++---- tests/e2e/logging/test_datadog_reader.py | 20 ++++++++++++++++ 3 files changed, 40 insertions(+), 14 deletions(-) create mode 100644 tests/e2e/logging/test_datadog_reader.py diff --git a/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py b/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py index 9a45743a0cd..75817340876 100644 --- a/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py +++ b/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py @@ -16,8 +16,10 @@ into a chat completion chunk. Two customer-visible contracts only hold on that p from __future__ import annotations +from typing import Final, Literal + import pytest -from pydantic import BaseModel +from pydantic import BaseModel, Field from e2e_config import unique_marker from e2e_http import StreamingResponse @@ -51,7 +53,8 @@ class _BridgeChoice(BaseModel): class _BridgeChunk(BaseModel): id: str - choices: list[_BridgeChoice] = [] + object: Literal["chat.completion.chunk"] + choices: list[_BridgeChoice] = Field(default_factory=list) class _WeatherArgs(BaseModel): @@ -103,16 +106,19 @@ class TestResponsesBridgeChatCompletionsStreaming: resources.key(), ChatBody( model=bridged_model, - messages=[ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}")], + messages=[ + ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}") + ], max_tokens=64, stream=True, ), ) - chunks = _bridge_chunks(result) - ids = {chunk.id for chunk in chunks} + chunks: Final = _bridge_chunks(result) + assert len(chunks) > 1, "the shared-id contract needs more than one streamed chunk" + ids: Final = frozenset(chunk.id for chunk in chunks) assert len(ids) == 1, f"bridged stream used {len(ids)} different chunk ids: {sorted(ids)[:5]}" - assert ids.pop().startswith("chatcmpl-"), f"bridged chunk id is not chat-completion shaped: {chunks[0].id}" + assert chunks[0].id.strip(), "bridged stream emitted an empty chunk id" @pytest.mark.covers( "llm.chat_completions.openai.basic.stream.bridge_streams_sse", @@ -134,9 +140,9 @@ class TestResponsesBridgeChatCompletionsStreaming: chunks = _bridge_chunks(result) content = "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) assert content.strip(), f"bridged stream completed with no content deltas: {result.stream_events[:3]}" - assert any( - choice.finish_reason for chunk in chunks for choice in chunk.choices - ), f"bridged stream never emitted a finish_reason: {result.stream_events[-3:]}" + assert any(choice.finish_reason for chunk in chunks for choice in chunk.choices), ( + f"bridged stream never emitted a finish_reason: {result.stream_events[-3:]}" + ) assert result.stream_done, f"bridged stream did not terminate with [DONE]: {result.stream_events[-2:]}" @pytest.mark.covers( diff --git a/tests/e2e/logging/datadog_reader.py b/tests/e2e/logging/datadog_reader.py index d0f478185c2..7b4372ef9ba 100644 --- a/tests/e2e/logging/datadog_reader.py +++ b/tests/e2e/logging/datadog_reader.py @@ -13,7 +13,7 @@ empty result. External reads go through ``e2e_http``. from __future__ import annotations import time -from dataclasses import dataclass +from dataclasses import dataclass, field import pytest from pydantic import BaseModel, ConfigDict, Field @@ -36,8 +36,8 @@ _RATE_LIMIT_RETRIES = 5 class _DdAuthHeaders(Headers): - api_key: str = Field(serialization_alias="DD-API-KEY") - app_key: str = Field(serialization_alias="DD-APPLICATION-KEY") + api_key: str = Field(serialization_alias="DD-API-KEY", repr=False) + app_key: str = Field(serialization_alias="DD-APPLICATION-KEY", repr=False) class _SearchFilter(BaseModel): @@ -88,8 +88,8 @@ class _SearchResponse(BaseModel): @dataclass(frozen=True, slots=True) class DdLogsReader: site: str - api_key: str - app_key: str + api_key: str = field(repr=False) + app_key: str = field(repr=False) def events_for_marker(self, marker: str) -> list[DdLogEvent]: """Every ingested event whose attributes carry the marker. DataDog diff --git a/tests/e2e/logging/test_datadog_reader.py b/tests/e2e/logging/test_datadog_reader.py new file mode 100644 index 00000000000..00624bd3c84 --- /dev/null +++ b/tests/e2e/logging/test_datadog_reader.py @@ -0,0 +1,20 @@ +from typing import Final + +from datadog_reader import DdLogsReader +from datadog_reader import _DdAuthHeaders # pyright: ignore[reportPrivateUsage] # verifies private auth-header serialization + + +def test_failure_diagnostics_hide_credentials_without_changing_auth_headers() -> None: + api_key: Final = "test-datadog-api-secret" + app_key: Final = "test-datadog-app-secret" + reader: Final = DdLogsReader(site="datadoghq.com", api_key=api_key, app_key=app_key) + headers: Final = _DdAuthHeaders(api_key=api_key, app_key=app_key) + + for value in (reader, headers): + assert api_key not in repr(value) + assert app_key not in repr(value) + + assert headers.model_dump(by_alias=True) == { + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + } From 80782d478ddbdd2e145b0cb72eb3c653efbe0ed0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:17:36 -0700 Subject: [PATCH 129/411] fix(policy_engine): keep legacy stream steps in sync and reject tool-call rewrites Each streaming step drops the response an earlier step's translation stored under request_data["response"], so a later legacy hook sees the stream as the steps before it left it instead of the first step's snapshot. A legacy replacement whose tool calls differ from the scanned chunks is now undeliverable like a text mismatch, so the original stream is released with a warning instead of delivering the text while dropping the tool-call change --- .../proxy/policy_engine/pipeline_executor.py | 17 ++-- .../policy_engine/test_pipeline_executor.py | 78 ++++++++++++++++--- 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 089f0683823..b4f6da18d1c 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -150,8 +150,8 @@ class _LegacyHookStreamAdapter(CustomGuardrail): non-streaming hooks, an exception it raises ends the stream through the executor's fail/error classification, and a replacement response is re-scanned by the same translation so its texts reach the client through the translation's ended-stream write-back. A - replacement whose scanned texts do not line up with the originals is undeliverable, so the - executor releases the original chunks.""" + replacement whose scanned texts do not line up with the originals, or whose tool calls + differ from them, is undeliverable, so the executor releases the original chunks.""" def __init__( self, @@ -184,8 +184,12 @@ class _LegacyHookStreamAdapter(CustomGuardrail): return inputs scanned: Final = _text_snapshot(inputs.get("texts")) rescanned: Final = await self._rescan(replacement, logging_obj) - rewritten: Final = None if rescanned is None else rescanned.get("texts") - if scanned is None or rewritten is None or len(rewritten) != len(scanned): + if scanned is None or rescanned is None: + raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") + rewritten: Final = rescanned.get("texts") + if rewritten is None or len(rewritten) != len(scanned): + raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") + if _tool_call_shapes(rescanned.get("tool_calls")) != _tool_call_shapes(inputs.get("tool_calls")): raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten} return rewritten_inputs @@ -380,7 +384,9 @@ class PipelineExecutor: yet (a tool-call rewrite, a text rewrite on a translation without write-back, or one the translation or adapter refused with ``UndeliverableStreamRewrite``) is discarded: the buffered chunks go back to the originals and the step passes, so the client gets - the stream the merge base sent.""" + the stream the merge base sent. The response an earlier step's translation stored under + ``request_data["response"]`` is dropped first, so this step's hook sees the stream as + the steps before it left it.""" scanner: Final = ( callback if PipelineExecutor.supports_unified_execution(callback) @@ -389,6 +395,7 @@ class PipelineExecutor: observer: Final = _StreamRewriteObserver(scanner) deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_text_rewrites originals: Final = copy.deepcopy(streaming_chunks) + hook_input.pop("response", None) # rebind-ok: an earlier step's stored response goes so this step's is stored try: if deliver_rewrites: await endpoint_translation.process_output_streaming_response( diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index a44afc6ef96..860f83a06bb 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -675,7 +675,6 @@ async def test_guardrail_not_found_uses_on_fail(monkeypatch): ], ) - result = await PipelineExecutor.execute_steps( steps=pipeline.steps, mode=pipeline.mode, @@ -1298,8 +1297,8 @@ async def test_streaming_step_restores_chunks_when_translation_refuses_the_rewri class _LegacyHookGuardrail(CustomGuardrail): """A guardrail with only the legacy post-call hook: it never defines apply_guardrail.""" - def __init__(self, replacement=None, raises=None): - super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True) + def __init__(self, replacement=None, raises=None, guardrail_name="masker"): + super().__init__(guardrail_name=guardrail_name, event_hook="post_call", default_on=True) self.replacement = replacement self.raises = raises self.calls = [] @@ -1339,7 +1338,7 @@ class _LegacyScanningTranslation: ): request_data.setdefault("response", {"text": responses_so_far[0]["text"]}) outputs = await guardrail_to_apply.apply_guardrail( - inputs={"texts": [responses_so_far[0]["text"]]}, + inputs={"texts": [responses_so_far[0]["text"]], "tool_calls": [dict(responses_so_far[0]["tool_call"])]}, request_data=request_data, input_type="response", logging_obj=litellm_logging_obj, @@ -1350,8 +1349,11 @@ class _LegacyScanningTranslation: async def process_output_response( self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None ): + inputs = {"texts": list(response["texts"])} + if response.get("tool_calls"): + inputs["tool_calls"] = list(response["tool_calls"]) await guardrail_to_apply.apply_guardrail( - inputs={"texts": list(response["texts"])}, + inputs=inputs, request_data={"response": response}, input_type="response", logging_obj=litellm_logging_obj, @@ -1359,10 +1361,26 @@ class _LegacyScanningTranslation: return response +def _legacy_replacement(*texts, tool_calls=None): + return {"texts": list(texts), "tool_calls": [_chunk()["tool_call"]] if tool_calls is None else tool_calls} + + async def _run_legacy_streaming_step(monkeypatch, guardrail, chunks, on_fail="block", on_error="next"): - monkeypatch.setattr(litellm, "callbacks", [guardrail]) + return await _run_legacy_streaming_steps(monkeypatch, [guardrail], chunks, on_fail=on_fail, on_error=on_error) + + +async def _run_legacy_streaming_steps(monkeypatch, guardrails, chunks, on_fail="block", on_error="next"): + monkeypatch.setattr(litellm, "callbacks", list(guardrails)) return await PipelineExecutor.execute_steps( - steps=[PipelineStep(guardrail="masker", on_pass="allow", on_fail=on_fail, on_error=on_error)], + steps=[ + PipelineStep( + guardrail=guardrail.guardrail_name, + on_pass="next" if position + 1 < len(guardrails) else "allow", + on_fail=on_fail, + on_error=on_error, + ) + for position, guardrail in enumerate(guardrails) + ], mode="post_call", data={"model": "m"}, user_api_key_dict=MagicMock(), @@ -1376,7 +1394,7 @@ async def _run_legacy_streaming_step(monkeypatch, guardrail, chunks, on_fail="bl @pytest.mark.asyncio @pytest.mark.parametrize("guardrail_class", [_LegacyHookGuardrail, _NativeHooksGuardrail]) async def test_streaming_step_runs_legacy_hook_and_delivers_its_rewrite(monkeypatch, caplog, guardrail_class): - guardrail = guardrail_class(replacement={"texts": ["[REWRITTEN] hello world"]}) + guardrail = guardrail_class(replacement=_legacy_replacement("[REWRITTEN] hello world")) chunks = [_chunk()] with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): @@ -1434,7 +1452,7 @@ async def test_streaming_step_takes_on_error_when_legacy_hook_crashes(monkeypatc @pytest.mark.asyncio async def test_streaming_step_discards_legacy_rewrite_whose_texts_do_not_line_up(monkeypatch, caplog): - guardrail = _LegacyHookGuardrail(replacement={"texts": ["split", "in two"]}) + guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement("split", "in two")) chunks = [_chunk()] with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): @@ -1442,3 +1460,45 @@ async def test_streaming_step_discards_legacy_rewrite_whose_texts_do_not_line_up _assert_passed_with_discard_warning(result, caplog) assert chunks == [_chunk()] + + +@pytest.mark.asyncio +async def test_streaming_step_discards_legacy_rewrite_that_changes_a_tool_call(monkeypatch, caplog): + masked_tool_call = {"function": {"name": "lookup", "arguments": '{"ssn": "[MASKED]"}'}} + guardrail = _LegacyHookGuardrail( + replacement=_legacy_replacement("[REWRITTEN] hello world", tool_calls=[masked_tool_call]) + ) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] + + +@pytest.mark.asyncio +async def test_streaming_step_discards_legacy_rewrite_that_drops_the_tool_calls(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world", tool_calls=[])) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] + + +@pytest.mark.asyncio +async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(monkeypatch): + masker = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world")) + auditor = _LegacyHookGuardrail(replacement=None, guardrail_name="auditor") + chunks = [_chunk()] + + result = await _run_legacy_streaming_steps(monkeypatch, [masker, auditor], chunks, on_fail="next") + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass", "pass"] + assert chunks[0]["text"] == "[REWRITTEN] hello world" + assert [call["response"] for call in masker.calls] == [{"native": True, "text": "hello world"}] + assert [call["response"] for call in auditor.calls] == [{"native": True, "text": "[REWRITTEN] hello world"}] From 64afa9d6eca8d238da02458a2769294eff93d08d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 15:36:34 -0700 Subject: [PATCH 130/411] test: isolate auto-router scenarios and clean partial setup --- .../test_auto_router_regressions_e2e.py | 239 +++++++++--------- 1 file changed, 120 insertions(+), 119 deletions(-) diff --git a/tests/e2e/router/test_auto_router_regressions_e2e.py b/tests/e2e/router/test_auto_router_regressions_e2e.py index 188db2a8eb5..374badcf5fc 100644 --- a/tests/e2e/router/test_auto_router_regressions_e2e.py +++ b/tests/e2e/router/test_auto_router_regressions_e2e.py @@ -41,6 +41,7 @@ which stores either the registered alias or the provider-prefixed form. import json import os from collections.abc import Iterator +from contextlib import ExitStack from dataclasses import dataclass from typing import Final @@ -120,19 +121,10 @@ class ResponsesApiResponse(BaseModel): @dataclass(frozen=True, slots=True) -class TagSplitDeployments: - """Scenario A mirrors the customer-shaped config from GitHub issue #36619: - plain deployment registered first, tier deployment and marker both tagged. - Scenario B flips both axes for GitHub issue #36621: marker registered first - and its tier deployment left untagged, so routing depends neither on - registration order nor on tier deployments carrying tags.""" - - tag_a: str - shared_a: str - tier_a: str - tag_b: str - shared_b: str - tier_b: str +class TagSplitDeployment: + tag: str + shared: str + tier: str @dataclass(frozen=True, slots=True) @@ -173,9 +165,7 @@ def _uniform_tier_config(tier_model: str) -> dict[str, object]: } -def _key_for( - proxy: ProxyClient, resources: ResourceManager, models: list[str], tag_filtering: bool = False -) -> str: +def _key_for(proxy: ProxyClient, resources: ResourceManager, models: list[str], tag_filtering: bool = False) -> str: key: Final = proxy.generate_key( KeyGenerateBody( models=models, @@ -211,46 +201,61 @@ def _assert_served_only_by(rows: list[SpendLogRow], allowed: frozenset[str], con ) -@pytest.fixture(scope="module") -def split(proxy: ProxyClient) -> Iterator[TagSplitDeployments]: +@pytest.fixture(scope="class") +def router_stack() -> Iterator[ExitStack]: + with ExitStack() as stack: + yield stack + + +def _register_models( + proxy: ProxyClient, stack: ExitStack, registrations: tuple[tuple[str, LiteLLMParamsBody], ...] +) -> None: + for name, params in registrations: + stack.callback(proxy.delete_model, proxy.create_model(name, params)) + + +def _tag_split(proxy: ProxyClient, stack: ExitStack, *, marker_first: bool) -> TagSplitDeployment: marker: Final = unique_marker() - deployments: Final = TagSplitDeployments( - tag_a=f"e2e-split-a-{marker}", - shared_a=f"e2e-autoroute-a-{marker}", - tier_a=f"e2e-tier-a-{marker}", - tag_b=f"e2e-split-b-{marker}", - shared_b=f"e2e-autoroute-b-{marker}", - tier_b=f"e2e-tier-b-{marker}", + named: Final = TagSplitDeployment( + tag=f"e2e-split-{marker}", + shared=f"e2e-autoroute-{marker}", + tier=f"e2e-tier-{marker}", ) anthropic_key: Final = _provider_key("ANTHROPIC_API_KEY") - marker_params_a: Final = LiteLLMParamsBody( - model="auto_router/complexity_router", - complexity_router_config=_uniform_tier_config(deployments.tier_a), - tags=[deployments.tag_a], + marker_registration: Final = ( + named.shared, + LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config=_uniform_tier_config(named.tier), + tags=[named.tag], + ), ) - marker_params_b: Final = LiteLLMParamsBody( - model="auto_router/complexity_router", - complexity_router_config=_uniform_tier_config(deployments.tier_b), - tags=[deployments.tag_b], + tier_registration: Final = ( + named.tier, + LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key, tags=None if marker_first else [named.tag]), ) - registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = ( - (deployments.shared_a, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)), - (deployments.tier_a, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key, tags=[deployments.tag_a])), - (deployments.shared_a, marker_params_a), - (deployments.shared_b, marker_params_b), - (deployments.tier_b, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key)), - (deployments.shared_b, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)), + plain_registration: Final = (named.shared, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)) + registrations: Final = ( + (marker_registration, tier_registration, plain_registration) + if marker_first + else (plain_registration, tier_registration, marker_registration) ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield deployments - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, stack, registrations) + return named -@pytest.fixture(scope="module") -def zero_priced_alias(proxy: ProxyClient) -> Iterator[ZeroPricedAlias]: +@pytest.fixture(scope="class") +def plain_first_split(proxy: ProxyClient, router_stack: ExitStack) -> TagSplitDeployment: + return _tag_split(proxy, router_stack, marker_first=False) + + +@pytest.fixture(scope="class") +def marker_first_split(proxy: ProxyClient, router_stack: ExitStack) -> TagSplitDeployment: + return _tag_split(proxy, router_stack, marker_first=True) + + +@pytest.fixture(scope="class") +def zero_priced_alias(proxy: ProxyClient, router_stack: ExitStack) -> ZeroPricedAlias: marker: Final = unique_marker() named: Final = ZeroPricedAlias(alias=f"e2e-priced-alias-{marker}", tier=f"e2e-priced-tier-{marker}") alias_params: Final = LiteLLMParamsBody( @@ -263,16 +268,12 @@ def zero_priced_alias(proxy: ProxyClient) -> Iterator[ZeroPricedAlias]: (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), (named.alias, alias_params), ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield named - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, router_stack, registrations) + return named -@pytest.fixture(scope="module") -def heuristic_split(proxy: ProxyClient) -> Iterator[HeuristicSplit]: +@pytest.fixture(scope="class") +def heuristic_split(proxy: ProxyClient, router_stack: ExitStack) -> HeuristicSplit: marker: Final = unique_marker() named: Final = HeuristicSplit( alias=f"e2e-heuristic-router-{marker}", @@ -289,16 +290,12 @@ def heuristic_split(proxy: ProxyClient) -> Iterator[HeuristicSplit]: (named.strong, LiteLLMParamsBody(model=STRONG_MODEL, api_key=_provider_key("OPENAI_API_KEY"))), (named.alias, LiteLLMParamsBody(model="auto_router/complexity_router", complexity_router_config=config)), ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield named - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, router_stack, registrations) + return named -@pytest.fixture(scope="module") -def semantic_auto_router(proxy: ProxyClient) -> Iterator[SemanticAutoRouter]: +@pytest.fixture(scope="class") +def semantic_auto_router(proxy: ProxyClient, router_stack: ExitStack) -> SemanticAutoRouter: marker: Final = unique_marker() named: Final = SemanticAutoRouter( marker=f"e2e-semantic-router-{marker}", @@ -321,16 +318,12 @@ def semantic_auto_router(proxy: ProxyClient) -> Iterator[SemanticAutoRouter]: (named.fallback, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), (named.marker, marker_params), ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield named - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, router_stack, registrations) + return named -@pytest.fixture(scope="module") -def credentialed_alias(proxy: ProxyClient) -> Iterator[CredentialedAlias]: +@pytest.fixture(scope="class") +def credentialed_alias(proxy: ProxyClient, router_stack: ExitStack) -> CredentialedAlias: marker: Final = unique_marker() named: Final = CredentialedAlias(alias=f"e2e-cred-alias-{marker}", tier=f"e2e-cred-tier-{marker}") alias_params: Final = LiteLLMParamsBody( @@ -342,104 +335,110 @@ def credentialed_alias(proxy: ProxyClient) -> Iterator[CredentialedAlias]: (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), (named.alias, alias_params), ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield named - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, router_stack, registrations) + return named class TestTagSplitRouting: @pytest.mark.covers("reliability.routing.tagged_marker.request_tag_selects_marker") def test_body_tagged_chat_routes_through_the_marker_to_its_tier( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins GitHub issue #36619: with tag filtering on, a chat request whose body metadata tags match the tagged marker under a shared model name is answered by the marker's tier deployment, not by the plain deployment that was registered under the name first.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) - chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a, tags=[split.tag_a]))) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) + chat: Final = unwrap(proxy.chat(key, _hello_chat_body(plain_first_split.shared, tags=[plain_first_split.tag]))) assert chat.choices, "tagged chat through the shared name returned no choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged chat on the shared name") + _assert_served_only_by(rows, CHEAP_SERVED | {plain_first_split.tier}, "body-tagged chat on the shared name") @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") def test_untagged_chat_is_always_served_by_the_plain_deployment( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins GitHub issue #36620: untagged chat requests to the shared name succeed on every call and are all served by the plain deployment; the tagged marker never captures them, so no intermittent auto-router errors and no tier hijacking.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) for _ in range(5): - chat = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a))) + chat = unwrap(proxy.chat(key, _hello_chat_body(plain_first_split.shared))) assert chat.choices, "untagged chat through the shared name returned no choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=5) - _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged chat on the shared name") + _assert_served_only_by(rows, PLAIN_SERVED | {plain_first_split.shared}, "untagged chat on the shared name") @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") def test_untagged_messages_is_served_by_the_plain_deployment( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins GitHub issue #36620 on the /v1/messages surface: an untagged Anthropic-native request to the shared name is served by the plain deployment, not captured by the tagged marker.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) - answer: Final = unwrap(proxy.messages(key, _hello_messages_body(split.shared_a))) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) + answer: Final = unwrap(proxy.messages(key, _hello_messages_body(plain_first_split.shared))) assert answer.content or answer.choices, "untagged /v1/messages returned neither content nor choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/messages on the shared name") + _assert_served_only_by( + rows, PLAIN_SERVED | {plain_first_split.shared}, "untagged /v1/messages on the shared name" + ) class TestUntaggedTierDeployments: @pytest.mark.covers("reliability.routing.tagged_marker.header_tag_selects_marker") def test_header_tagged_messages_routes_through_the_marker_to_an_untagged_tier( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, marker_first_split: TagSplitDeployment ) -> None: """Pins GitHub issue #36621: a /v1/messages request tagged only via the x-litellm-tags header selects the tagged marker, and the rewrite still lands on the tier deployment even though that deployment carries no tags, because the marker consumed the routing tags.""" - key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True) - headers: Final = TaggedAnthropicHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_b) + key: Final = _key_for( + proxy, resources, [marker_first_split.shared, marker_first_split.tier], tag_filtering=True + ) + headers: Final = TaggedAnthropicHeaders(authorization=f"Bearer {key}", x_litellm_tags=marker_first_split.tag) answer: Final = unwrap( proxy.transport.post( "/v1/messages", headers=headers, - json=_hello_messages_body(split.shared_b), + json=_hello_messages_body(marker_first_split.shared), response_type=AnthropicMessagesResponse, ) ) assert answer.content or answer.choices, "header-tagged /v1/messages returned neither content nor choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "header-tagged /v1/messages on the shared name") + _assert_served_only_by( + rows, CHEAP_SERVED | {marker_first_split.tier}, "header-tagged /v1/messages on the shared name" + ) @pytest.mark.covers("reliability.routing.tagged_marker.untagged_tier_deployments_still_served") def test_body_tagged_chat_reaches_the_untagged_tier_after_marker_rewrite( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, marker_first_split: TagSplitDeployment ) -> None: """Pins the tag-consumption half of GitHub issue #36621: after the tagged marker rewrites the request to its tier model, the consumed routing tags no longer constrain deployment selection, so the untagged tier deployment serves the request instead of a strict-tag denial.""" - key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True) - chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_b, tags=[split.tag_b]))) + key: Final = _key_for( + proxy, resources, [marker_first_split.shared, marker_first_split.tier], tag_filtering=True + ) + chat: Final = unwrap( + proxy.chat(key, _hello_chat_body(marker_first_split.shared, tags=[marker_first_split.tag])) + ) assert chat.choices, "body-tagged chat through the marker-first shared name returned no choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "body-tagged chat with untagged tier") + _assert_served_only_by(rows, CHEAP_SERVED | {marker_first_split.tier}, "body-tagged chat with untagged tier") @pytest.mark.covers("reliability.routing.tagged_marker.tag_semantics_stay_strict") def test_tagged_call_straight_at_an_untagged_deployment_stays_denied( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, marker_first_split: TagSplitDeployment ) -> None: """The tag-consumption fix must not loosen strict tag semantics: a tagged request aimed directly at an untagged deployment (no marker involved) is still rejected with the 401 tags-configuration error.""" - key: Final = _key_for(proxy, resources, [split.tier_b], tag_filtering=True) - result: Final = proxy.chat(key, _hello_chat_body(split.tier_b, tags=[split.tag_b])) + key: Final = _key_for(proxy, resources, [marker_first_split.tier], tag_filtering=True) + result: Final = proxy.chat(key, _hello_chat_body(marker_first_split.tier, tags=[marker_first_split.tag])) assert isinstance(result, UnauthorizedError), ( f"expected the tagged direct call to an untagged deployment to be denied with 401, got {result}" ) @@ -451,37 +450,39 @@ class TestUntaggedTierDeployments: class TestResponsesApiTagRouting: @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker") def test_header_tagged_responses_with_string_input_routes_to_the_tier( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins the /v1/responses surface of the tag split (GitHub issues #36620/#36621): a /v1/responses request with string input, tagged via the x-litellm-tags header, succeeds and routes through the tagged marker to its tier.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) - headers: Final = TaggedAuthHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_a) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) + headers: Final = TaggedAuthHeaders(authorization=f"Bearer {key}", x_litellm_tags=plain_first_split.tag) body: Final = ResponsesBody( - model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64 + model=plain_first_split.shared, input=f"say hello {unique_marker()}", max_output_tokens=64 ) answer: Final = unwrap( proxy.transport.post("/v1/responses", headers=headers, json=body, response_type=ResponsesApiResponse) ) assert answer.id, "header-tagged /v1/responses returned no response id" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "header-tagged /v1/responses string input") + _assert_served_only_by( + rows, CHEAP_SERVED | {plain_first_split.tier}, "header-tagged /v1/responses string input" + ) @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker") def test_body_tagged_responses_with_list_input_routes_to_the_tier( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins the body-tag and list-input combination of the same split: /v1/responses with litellm_metadata.tags and structured input items routes through the tagged marker to its tier.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) body: Final = ResponsesBody( - model=split.shared_a, + model=plain_first_split.shared, input=[ResponsesInputItem(role="user", content=f"say hello {unique_marker()}")], max_output_tokens=64, - litellm_metadata=ResponsesTagMetadata(tags=[split.tag_a]), + litellm_metadata=ResponsesTagMetadata(tags=[plain_first_split.tag]), ) answer: Final = unwrap( proxy.transport.post( @@ -493,18 +494,18 @@ class TestResponsesApiTagRouting: ) assert answer.id, "body-tagged /v1/responses returned no response id" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged /v1/responses list input") + _assert_served_only_by(rows, CHEAP_SERVED | {plain_first_split.tier}, "body-tagged /v1/responses list input") @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") def test_untagged_responses_is_served_by_the_plain_deployment( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins the untagged half of the /v1/responses tag split: an untagged request to the shared name is served by the plain deployment, matching the chat and messages surfaces.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) body: Final = ResponsesBody( - model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64 + model=plain_first_split.shared, input=f"say hello {unique_marker()}", max_output_tokens=64 ) answer: Final = unwrap( proxy.transport.post( @@ -516,7 +517,9 @@ class TestResponsesApiTagRouting: ) assert answer.id, "untagged /v1/responses returned no response id" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/responses on the shared name") + _assert_served_only_by( + rows, PLAIN_SERVED | {plain_first_split.shared}, "untagged /v1/responses on the shared name" + ) class TestStrategyAliasPricing: @@ -551,9 +554,7 @@ class TestComplexityHeuristicScope: while the accompanying ~2KB agent system prompt is packed with enough reasoning and complexity keywords that scoring the combined text lands in REASONING; only ask-only scoring keeps this on the cheap tier.""" - key: Final = _key_for( - proxy, resources, [heuristic_split.alias, heuristic_split.cheap, heuristic_split.strong] - ) + key: Final = _key_for(proxy, resources, [heuristic_split.alias, heuristic_split.cheap, heuristic_split.strong]) body: Final = ChatBody( model=heuristic_split.alias, messages=[ From 71a4a6e912624c31fde159fad5f6508e6b2a217a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:20:50 -0700 Subject: [PATCH 131/411] refactor: keep the AWS JSON signing helper under llms and bound the probe's refresh wait --- litellm/llms/bedrock/base_aws_llm.py | 19 ++++++++++++ .../llm_passthrough_endpoints.py | 29 +++---------------- .../llms/bedrock/event_loop_probe.py | 5 ++++ 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index bbd4988b204..1c7268ffa22 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -1671,6 +1671,25 @@ class BaseAWSLLM: return request_headers_dict, request.body +def sign_aws_json_post( + get_credentials: Callable[[], Credentials], + service_name: str, + aws_region_name: str | None, + url: str, + body: str, + headers: Mapping[str, str], +) -> AWSPreparedRequest: + try: + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + except ImportError: + raise ImportError(f"Missing boto3 to call {service_name}. Run 'pip install boto3'.") + + aws_request: Final = AWSRequest(method="POST", url=url, data=body, headers=headers) + SigV4Auth(get_credentials(), service_name, aws_region_name).add_auth(aws_request) + return aws_request.prepare() + + _SignParams = ParamSpec("_SignParams") _SignedRequest = TypeVar("_SignedRequest") diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 6085bc2bfce..037f663044f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -86,9 +86,6 @@ from litellm.utils import ProviderConfigManager from .passthrough_endpoint_router import PassthroughEndpointRouter if TYPE_CHECKING: - from botocore.awsrequest import AWSPreparedRequest - from botocore.credentials import Credentials - from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig from litellm.router import Router @@ -1134,6 +1131,7 @@ async def bedrock_proxy_route( ) # Add or update query parameters + from litellm.llms.bedrock.base_aws_llm import sign_aws_json_post from litellm.llms.bedrock.chat import BedrockConverseLLM bedrock_llm: Final = BedrockConverseLLM() @@ -1143,7 +1141,7 @@ async def bedrock_proxy_route( except Exception as e: raise HTTPException(status_code=400, detail={"error": e}) prepped: Final = await asyncio.to_thread( - _sign_aws_json_post, + sign_aws_json_post, get_credentials=bedrock_llm.get_credentials, service_name="bedrock", aws_region_name=aws_region_name, @@ -1178,25 +1176,6 @@ async def bedrock_proxy_route( return received_value -def _sign_aws_json_post( - get_credentials: Callable[[], Credentials], - service_name: str, - aws_region_name: str | None, - url: str, - body: str, - headers: Mapping[str, str], -) -> AWSPreparedRequest: - try: - from botocore.auth import SigV4Auth - from botocore.awsrequest import AWSRequest - except ImportError: - raise ImportError(f"Missing boto3 to call {service_name}. Run 'pip install boto3'.") - - aws_request: Final = AWSRequest(method="POST", url=url, data=body, headers=headers) - SigV4Auth(get_credentials(), service_name, aws_region_name).add_auth(aws_request) - return aws_request.prepare() - - COMPREHEND_MEDICAL_TARGET_PREFIX: Final = "ComprehendMedical_20181030" @@ -1257,11 +1236,11 @@ async def comprehend_medical_proxy_route( if "stream" in data: raise HTTPException(status_code=400, detail="'stream' is not a Comprehend Medical request member") - from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, sign_aws_json_post target_url: Final = f"https://comprehendmedical.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/" prepped: Final = await asyncio.to_thread( - _sign_aws_json_post, + sign_aws_json_post, get_credentials=partial(BaseAWSLLM().get_credentials, aws_region_name=aws_region_name), service_name="comprehendmedical", aws_region_name=aws_region_name, diff --git a/tests/test_litellm/llms/bedrock/event_loop_probe.py b/tests/test_litellm/llms/bedrock/event_loop_probe.py index 68acce308b8..c347247ec32 100644 --- a/tests/test_litellm/llms/bedrock/event_loop_probe.py +++ b/tests/test_litellm/llms/bedrock/event_loop_probe.py @@ -4,12 +4,14 @@ from __future__ import annotations import asyncio import threading +import time from datetime import datetime, timedelta, timezone from typing import Final from botocore.credentials import RefreshableCredentials REFRESH_RELEASE_TIMEOUT_SECONDS: Final = 2.0 +REFRESH_START_TIMEOUT_SECONDS: Final = 10.0 class EventLoopProbe: @@ -47,6 +49,9 @@ class EventLoopProbe: ) async def release_refresh_from_the_loop(self) -> None: + deadline: Final = time.monotonic() + REFRESH_START_TIMEOUT_SECONDS while not self.refresh_started.is_set(): + if time.monotonic() > deadline: + raise TimeoutError("signing finished without ever starting a credential refresh") await asyncio.sleep(0.005) self.loop_served.set() From 3ea65e761ee2f021e967c87202267cd4d70ff3f2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:30:06 -0700 Subject: [PATCH 132/411] test(guardrails): cover Responses and Messages pipeline tool-call delivery --- .../proxy_logging/test_guardrail_pipeline.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index f753d2ab690..bf71780ad7b 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -25,6 +25,7 @@ from litellm.integrations.custom_guardrail import ( ModifyResponseException, ) from litellm.integrations.prometheus import PrometheusLogger +from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines @@ -2185,3 +2186,106 @@ async def test_per_chunk_streaming_hook_runs_guardrail_whose_pipeline_cannot_str assert result is not None assert seen["count"] == 1 assert seen["response"] == "hello " + + +def _mask_tool_call_arguments(inputs: Dict[str, Any]) -> Dict[str, Any]: + return { + "tool_calls": [ + { + "id": stream_item_field(tool_call, "id"), + "type": "function", + "function": { + "name": stream_item_field(stream_item_field(tool_call, "function"), "name"), + "arguments": '{"fruit": "[MASKED]"}', + }, + } + for tool_call in inputs.get("tool_calls", []) + ] + } + + +def _anthropic_tool_use_sse_chunks() -> List[bytes]: + events = [ + ("message_start", {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "m", "content": [], "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 0}}}), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup_fruit", "input": {}}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"fruit": "persim'}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": 'mon"}'}}), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, "usage": {"output_tokens": 2}}), + ("message_stop", {"type": "message_stop"}), + ] + return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events] + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_delivers_tool_use_rewrite_on_anthropic_sse( + proxy_logging, make_user_api_key_auth, monkeypatch +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_mask_tool_call_arguments)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"), + response=_async_chunk_iter(_anthropic_tool_use_sse_chunks()), + request_data=data, + ) + ] + + raw = b"".join(delivered).decode() + assert '{\\"fruit\\": \\"[MASKED]\\"}' in raw + assert "persim" not in raw + assert '"name": "lookup_fruit"' in raw and '"id": "toolu_1"' in raw + assert '"stop_reason": "tool_use"' in raw + assert raw.count("event: content_block_delta") == 2 + + +def _responses_function_call_events() -> List[Dict[str, Any]]: + def item(arguments: str, status: str) -> Dict[str, Any]: + return { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "lookup_fruit", + "arguments": arguments, + "status": status, + } + + return [ + {"type": "response.output_item.added", "output_index": 0, "item": item("", "in_progress")}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_1", "output_index": 0, "delta": '{"fruit":'}, + {"type": "response.function_call_arguments.delta", "item_id": "fc_1", "output_index": 0, "delta": ' "persimmon"}'}, + {"type": "response.function_call_arguments.done", "item_id": "fc_1", "output_index": 0, "arguments": '{"fruit": "persimmon"}'}, + {"type": "response.output_item.done", "output_index": 0, "item": item('{"fruit": "persimmon"}', "completed")}, + { + "type": "response.completed", + "response": {"id": "resp_1", "created_at": 1, "model": "m", "output": [item('{"fruit": "persimmon"}', "completed")], "status": "completed"}, + }, + ] + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_delivers_function_call_rewrite_on_responses_events( + proxy_logging, make_user_api_key_auth, monkeypatch +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_mask_tool_call_arguments)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/responses"), + response=_async_chunk_iter(_responses_function_call_events()), + request_data=data, + ) + ] + + assert [event["type"] for event in delivered] == [event["type"] for event in _responses_function_call_events()] + assert [event["delta"] for event in delivered if event["type"] == "response.function_call_arguments.delta"] == ['{"fruit": "[MASKED]"}', ""] + assert delivered[3]["arguments"] == '{"fruit": "[MASKED]"}' + assert delivered[4]["item"]["arguments"] == '{"fruit": "[MASKED]"}' + assert delivered[5]["response"]["output"][0]["arguments"] == '{"fruit": "[MASKED]"}' + assert "persimmon" not in json.dumps(delivered) From 7a6c0cbf08d9c564697c4df7d70d661b0e6241b9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:43:28 -0700 Subject: [PATCH 133/411] fix(spend-tracking): drop the owner of a digest shared by several users and back off failed scans --- .../spend_tracking/key_metadata_recovery.py | 54 +++++++++++-------- .../test_key_metadata_recovery.py | 49 +++++++++++++++-- 2 files changed, 79 insertions(+), 24 deletions(-) diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index 319050b87cd..d555da99397 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -38,20 +38,23 @@ ORDER BY token, deleted_at DESC _SPEND_LOG_ALIAS_SQL: Final = """ SELECT DISTINCT ON (api_key) api_key AS digest, - metadata->>'user_api_key_alias' AS key_alias, - COALESCE(NULLIF(team_id, ''), metadata->>'user_api_key_team_id') AS team_id, - COALESCE(NULLIF("user", ''), metadata->>'user_api_key_user_id') AS user_id -FROM "LiteLLM_SpendLogs" -WHERE api_key = ANY($1::text[]) - AND "startTime" >= $2::timestamp - AND "startTime" < $3::timestamp - AND COALESCE( - metadata->>'user_api_key_alias', - NULLIF("user", ''), - metadata->>'user_api_key_user_id', - NULLIF(team_id, ''), - metadata->>'user_api_key_team_id' - ) IS NOT NULL + key_alias, + team_id, + user_id, + MIN(user_id) OVER (PARTITION BY api_key) AS first_owner, + MAX(user_id) OVER (PARTITION BY api_key) AS last_owner +FROM ( + SELECT api_key, + "startTime", + NULLIF(metadata->>'user_api_key_alias', '') AS key_alias, + COALESCE(NULLIF(team_id, ''), NULLIF(metadata->>'user_api_key_team_id', '')) AS team_id, + COALESCE(NULLIF("user", ''), NULLIF(metadata->>'user_api_key_user_id', '')) AS user_id + FROM "LiteLLM_SpendLogs" + WHERE api_key = ANY($1::text[]) + AND "startTime" >= $2::timestamp + AND "startTime" < $3::timestamp +) named +WHERE COALESCE(key_alias, user_id, team_id) IS NOT NULL ORDER BY api_key, "startTime" DESC """ @@ -72,7 +75,16 @@ class _TokenDigestRow(BaseModel): user_id: str | None = None +class _SpendLogDigestRow(_TokenDigestRow): + first_owner: str | None = None + last_owner: str | None = None + + def unanimous_owner(self) -> str | None: + return self.user_id if self.first_owner == self.last_owner else None + + _TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...]) +_SPEND_LOG_DIGEST_ROWS: Final = TypeAdapter(tuple[_SpendLogDigestRow, ...]) _CACHED_KEY_METADATA: Final = TypeAdapter(KeyMetadataDict) _SPEND_LOG_METADATA_CACHE: Final = InMemoryCache( max_size_in_memory=SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS, @@ -235,9 +247,10 @@ async def _query_spend_log_metadata( return None return MappingProxyType( { - row.digest: KeyMetadataDict(key_alias=row.key_alias, team_id=row.team_id, user_id=row.user_id) - for row in _TOKEN_DIGEST_ROWS.validate_python(rows) - if row.digest in digests and (row.key_alias or row.user_id or row.team_id) + row.digest: KeyMetadataDict(key_alias=row.key_alias, team_id=row.team_id, user_id=owner) + for row in _SPEND_LOG_DIGEST_ROWS.validate_python(rows) + for owner in (row.unanimous_owner(),) + if row.digest in digests and (row.key_alias or owner or row.team_id) } ) @@ -270,11 +283,10 @@ async def _spend_log_metadata_one_query_at_a_time( fresh: Final = ( await _query_spend_log_metadata(prisma_client, pending, window) if pending else _EMPTY_KEY_METADATA ) - if fresh is None: - return settled + found: Final = fresh if fresh is not None else _EMPTY_KEY_METADATA for digest in pending: - _remember_spend_log_metadata(cache, digest, window, fresh.get(digest)) - return MappingProxyType({**settled, **fresh}) + _remember_spend_log_metadata(cache, digest, window, found.get(digest)) + return MappingProxyType({**settled, **found}) async def recover_key_metadata_from_spend_logs( diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 5b45777502a..682b11f5bb7 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -386,20 +386,63 @@ async def test_recover_key_metadata_from_spend_logs_rescans_when_the_window_chan @pytest.mark.asyncio -async def test_recover_key_metadata_from_spend_logs_does_not_cache_a_failed_query(): +async def test_recover_key_metadata_from_spend_logs_retries_a_failed_query_only_after_the_miss_ttl(): digest = hash_token("cli-session-retry") window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) - cache = InMemoryCache() + cache = InMemoryCache(default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL) mock_prisma = MagicMock() - mock_prisma.db.query_raw = AsyncMock(side_effect=PrismaError("db down")) + mock_prisma.db.query_raw = AsyncMock(side_effect=PrismaError("statement timeout")) + started = time.time() assert await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) == {} mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(digest, "back-online", None, None)]) + assert await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) == {} + mock_prisma.db.query_raw.assert_not_awaited() + miss_key = next(key for key in cache.ttl_dict if digest in key and not key.endswith(":missed-before")) + assert cache.ttl_dict[miss_key] - started <= SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL + 1 + cache.ttl_dict[miss_key] = time.time() - 1 + result = await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) assert result[digest]["key_alias"] == "back-online" +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_drops_the_owner_of_a_digest_shared_by_several_users(): + shared_ui_digest = hash_token("ui-token") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs( + [ + { + **_digest_row(shared_ui_digest, "ui-token", "litellm-dashboard", "bob"), + "first_owner": "alice", + "last_owner": "bob", + } + ] + ) + + result = await recover_key_metadata_from_spend_logs( + mock_prisma, {shared_ui_digest}, window, cache=InMemoryCache() + ) + + assert result[shared_ui_digest] == {"key_alias": "ui-token", "team_id": "litellm-dashboard", "user_id": None} + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_keeps_the_owner_when_every_named_row_agrees(): + digest = hash_token("cli-session-one-owner") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + mock_prisma.db.query_raw = _query_raw_spend_logs( + [{**_digest_row(digest, None, None, "carol"), "first_owner": "carol", "last_owner": "carol"}] + ) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=InMemoryCache()) + + assert result[digest]["user_id"] == "carol" + + @pytest.mark.asyncio async def test_recover_key_metadata_from_spend_logs_forgets_a_miss_long_before_a_hit(): found = hash_token("cli-session-found") From ea427e33d8208034f934e52a7293f7f0bfa020ec Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:44:14 -0700 Subject: [PATCH 134/411] fix(policy_engine): deliver legacy stream rewrites made in place A legacy post-call hook that changes the response it was handed and returns None (the model armor guardrail masks choice content that way) used to have that rewrite dropped on the streaming pipeline path, since the adapter only re-scanned a returned replacement. The adapter now re-scans the response it handed the hook when the hook returns None, so an in-place rewrite reaches the client through the same ended-stream write-back --- .../proxy/policy_engine/pipeline_executor.py | 13 ++++--- .../policy_engine/test_pipeline_executor.py | 37 +++++++++++++++---- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index b4f6da18d1c..45a113fcda2 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -148,8 +148,9 @@ class _LegacyHookStreamAdapter(CustomGuardrail): endpoint translation hands it the texts it scanned plus the assembled response under ``request_data["response"]``; the hook gets that response in the shape its route gives non-streaming hooks, an exception it raises ends the stream through the executor's - fail/error classification, and a replacement response is re-scanned by the same translation - so its texts reach the client through the translation's ended-stream write-back. A + fail/error classification, and the response it hands back, or the one it changed in place + and returned ``None`` for, is re-scanned by the same translation so its texts reach the + client through the translation's ended-stream write-back. A replacement whose scanned texts do not line up with the originals, or whose tool calls differ from them, is undeliverable, so the executor releases the original chunks.""" @@ -175,15 +176,17 @@ class _LegacyHookStreamAdapter(CustomGuardrail): input_type: Literal["request", "response"], logging_obj: "LiteLLMLoggingObj | None" = None, ) -> GenericGuardrailAPIInputs: + hooked: Final = self.endpoint_translation.post_call_hook_response(request_data.get("response")) replacement: Final = await self.inner.async_post_call_success_hook( data=request_data, user_api_key_dict=self.user_api_key_dict, - response=self.endpoint_translation.post_call_hook_response(request_data.get("response")), + response=hooked, ) - if replacement is None: + rewrite: Final = hooked if replacement is None else replacement + if rewrite is None: return inputs scanned: Final = _text_snapshot(inputs.get("texts")) - rescanned: Final = await self._rescan(replacement, logging_obj) + rescanned: Final = await self._rescan(rewrite, logging_obj) if scanned is None or rescanned is None: raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") rewritten: Final = rescanned.get("texts") diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 860f83a06bb..ae7754e5821 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1297,16 +1297,19 @@ async def test_streaming_step_restores_chunks_when_translation_refuses_the_rewri class _LegacyHookGuardrail(CustomGuardrail): """A guardrail with only the legacy post-call hook: it never defines apply_guardrail.""" - def __init__(self, replacement=None, raises=None, guardrail_name="masker"): + def __init__(self, replacement=None, raises=None, guardrail_name="masker", rewrite_in_place=None): super().__init__(guardrail_name=guardrail_name, event_hook="post_call", default_on=True) self.replacement = replacement self.raises = raises + self.rewrite_in_place = rewrite_in_place self.calls = [] async def async_post_call_success_hook(self, data, user_api_key_dict, response): self.calls.append({"data": data, "user_api_key_dict": user_api_key_dict, "response": response}) if self.raises is not None: raise self.raises + if self.rewrite_in_place is not None: + response["text"] = self.rewrite_in_place return self.replacement @@ -1325,7 +1328,7 @@ class _LegacyScanningTranslation: delivers_ended_stream_text_rewrites = True def post_call_hook_response(self, response): - return {"native": True, "text": response["text"]} + return {"native": True, "text": response["text"], "tool_calls": response["tool_calls"]} async def process_output_streaming_response( self, @@ -1336,7 +1339,9 @@ class _LegacyScanningTranslation: request_data=None, deliver_ended_stream_rewrites=False, ): - request_data.setdefault("response", {"text": responses_so_far[0]["text"]}) + request_data.setdefault( + "response", {"text": responses_so_far[0]["text"], "tool_calls": [dict(responses_so_far[0]["tool_call"])]} + ) outputs = await guardrail_to_apply.apply_guardrail( inputs={"texts": [responses_so_far[0]["text"]], "tool_calls": [dict(responses_so_far[0]["tool_call"])]}, request_data=request_data, @@ -1349,7 +1354,7 @@ class _LegacyScanningTranslation: async def process_output_response( self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None ): - inputs = {"texts": list(response["texts"])} + inputs = {"texts": [response["text"]] if "text" in response else list(response["texts"])} if response.get("tool_calls"): inputs["tool_calls"] = list(response["tool_calls"]) await guardrail_to_apply.apply_guardrail( @@ -1361,6 +1366,10 @@ class _LegacyScanningTranslation: return response +def _native(text): + return {"native": True, "text": text, "tool_calls": [_chunk()["tool_call"]]} + + def _legacy_replacement(*texts, tool_calls=None): return {"texts": list(texts), "tool_calls": [_chunk()["tool_call"]] if tool_calls is None else tool_calls} @@ -1403,12 +1412,26 @@ async def test_streaming_step_runs_legacy_hook_and_delivers_its_rewrite(monkeypa assert result.terminal_action == "allow" assert [step.outcome for step in result.step_results] == ["pass"] assert chunks[0]["text"] == "[REWRITTEN] hello world" - assert [call["response"] for call in guardrail.calls] == [{"native": True, "text": "hello world"}] + assert [call["response"] for call in guardrail.calls] == [_native("hello world")] assert guardrail.calls[0]["data"]["model"] == "m" assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] assert not any("discarded" in record.getMessage() for record in caplog.records) +@pytest.mark.asyncio +async def test_streaming_step_delivers_a_legacy_rewrite_made_in_place(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(rewrite_in_place="[REWRITTEN] hello world") + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert chunks[0]["text"] == "[REWRITTEN] hello world" + assert not any("discarded" in record.getMessage() for record in caplog.records) + + @pytest.mark.asyncio async def test_streaming_step_passes_untouched_when_legacy_hook_returns_none(monkeypatch, caplog): guardrail = _LegacyHookGuardrail(replacement=None) @@ -1500,5 +1523,5 @@ async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(mon assert result.terminal_action == "allow" assert [step.outcome for step in result.step_results] == ["pass", "pass"] assert chunks[0]["text"] == "[REWRITTEN] hello world" - assert [call["response"] for call in masker.calls] == [{"native": True, "text": "hello world"}] - assert [call["response"] for call in auditor.calls] == [{"native": True, "text": "[REWRITTEN] hello world"}] + assert [call["response"] for call in masker.calls] == [_native("hello world")] + assert [call["response"] for call in auditor.calls] == [_native("[REWRITTEN] hello world")] From ce7cec1a3635bdb66292daa45ec9fe39f37bbffa Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:48:22 -0700 Subject: [PATCH 135/411] fix(policy_engine): keep discarded stream rewrites out of the applied-guardrails header A streaming step whose rewrite the executor threw away (a tool-call rewrite, a text rewrite the translation cannot write back, or one the adapter refused) still marked its guardrail as applied, so the header claimed an output the client never received. The step now returns right after releasing the original chunks, which leaves the header as the merge base sent it --- litellm/proxy/policy_engine/pipeline_executor.py | 10 ++++++---- .../proxy/policy_engine/test_pipeline_executor.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 45a113fcda2..9b0d1e60839 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -387,7 +387,8 @@ class PipelineExecutor: yet (a tool-call rewrite, a text rewrite on a translation without write-back, or one the translation or adapter refused with ``UndeliverableStreamRewrite``) is discarded: the buffered chunks go back to the originals and the step passes, so the client gets - the stream the merge base sent. The response an earlier step's translation stored under + the stream the merge base sent, and the guardrail stays out of the applied-guardrails + header since its output never reached the client. The response an earlier step's translation stored under ``request_data["response"]`` is dropped first, so this step's hook sees the stream as the steps before it left it.""" scanner: Final = ( @@ -419,9 +420,10 @@ class PipelineExecutor: ) except UndeliverableStreamRewrite: _release_original_chunks(step.guardrail, streaming_chunks, originals) - else: - if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites): - _release_original_chunks(step.guardrail, streaming_chunks, originals) + return + if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites): + _release_original_chunks(step.guardrail, streaming_chunks, originals) + return if not callback.records_own_guardrail_information: add_guardrail_to_applied_guardrails_header(request_data=hook_input, guardrail_name=step.guardrail) diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index ae7754e5821..2af318c88cd 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1146,6 +1146,7 @@ def _assert_passed_with_discard_warning(result, caplog): assert result.terminal_action == "allow" assert [step.outcome for step in result.step_results] == ["pass"] assert any("'masker'" in record.getMessage() and "discarded" in record.getMessage() for record in caplog.records) + assert "masker" not in ((result.modified_data or {}).get("metadata") or {}).get("applied_guardrails", []) @pytest.mark.asyncio From 253600fc6144747c1ffd954c3ebdfadd686adc2e Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 16:52:56 -0700 Subject: [PATCH 136/411] test: wait for requested guardrail propagation --- tests/e2e/guardrails/guardrails_client.py | 22 ++++++- .../e2e/guardrails/test_guardrails_client.py | 63 +++++++++++++++++++ .../test_tool_permission_guardrail_e2e.py | 24 ++++--- 3 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 tests/e2e/guardrails/test_guardrails_client.py diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 1f55a0f9a56..3770de0c6d5 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -7,7 +7,7 @@ from __future__ import annotations import time from collections.abc import Callable from dataclasses import dataclass -from typing import Literal +from typing import Final, Literal from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation, unique_marker from e2e_http import NoBody, Result, StreamingResponse, Success, unwrap @@ -405,6 +405,26 @@ def build_client(proxy: ProxyClient) -> GuardrailsClient: return GuardrailsClient(proxy=proxy) +def poll_until_guardrail_applied( + call: Callable[[], StreamingResponse], + guardrail_name: str, + *, + timeout: float = POLL_TIMEOUT, + interval: float = POLL_INTERVAL, + now: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> StreamingResponse: + deadline: Final = now() + timeout + while ( + (result := call()).ok + and guardrail_name + not in (name.strip() for name in result.headers.get("x-litellm-applied-guardrails", "").split(",")) + and (remaining := deadline - now()) > 0 + ): + sleep(min(interval, remaining)) + return result + + def poll_until_blocked[R: BaseModel](call: Callable[[], Result[R]]) -> Result[R]: """Retry a call that a guardrail should reject until it is, returning the last result. diff --git a/tests/e2e/guardrails/test_guardrails_client.py b/tests/e2e/guardrails/test_guardrails_client.py new file mode 100644 index 00000000000..aab163d86d2 --- /dev/null +++ b/tests/e2e/guardrails/test_guardrails_client.py @@ -0,0 +1,63 @@ +from dataclasses import dataclass +from itertools import chain, repeat +from typing import Final + +import pytest + +from e2e_http import StreamingResponse +from guardrails_client import poll_until_guardrail_applied + + +@dataclass +class Clock: + elapsed: float = 0.0 + + def now(self) -> float: + return self.elapsed + + def sleep(self, seconds: float) -> None: + self.elapsed += seconds + + +def _response(applied: str, status: int = 200) -> StreamingResponse: + return StreamingResponse(status_code=status, body="{}", headers={"x-litellm-applied-guardrails": applied}) + + +def test_waits_for_requested_guardrail_after_an_unrelated_global_guardrail() -> None: + clock: Final = Clock() + expected: Final = _response("global-filter, tool-permission") + responses: Final = iter((_response("global-filter"), expected)) + + result: Final = poll_until_guardrail_applied( + lambda: next(responses), "tool-permission", timeout=5, interval=2, now=clock.now, sleep=clock.sleep + ) + + assert result is expected + assert clock.elapsed == 2 + + +@pytest.mark.parametrize("applied", ("", "global-filter", "tool-permission-sibling")) +def test_missing_exact_guardrail_returns_failure_evidence_at_deadline(applied: str) -> None: + clock: Final = Clock() + missing: Final = _response(applied) + + result: Final = poll_until_guardrail_applied( + lambda: missing, "tool-permission", timeout=5, interval=2, now=clock.now, sleep=clock.sleep + ) + + assert result is missing + assert clock.elapsed == 5 + + +@pytest.mark.parametrize("status", (400, 401, 429, 500)) +def test_http_failure_is_not_hidden_by_a_later_success(status: int) -> None: + clock: Final = Clock() + failed: Final = _response("", status) + responses: Final = iter(chain((failed,), repeat(_response("tool-permission")))) + + result: Final = poll_until_guardrail_applied( + lambda: next(responses), "tool-permission", timeout=5, interval=2, now=clock.now, sleep=clock.sleep + ) + + assert result is failed + assert clock.elapsed == 0 diff --git a/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py index 9ef3650625c..8d1047e53c7 100644 --- a/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py @@ -30,6 +30,7 @@ from guardrails_client import ( ToolPermissionParamsBody, ToolPermissionRuleBody, poll_until_blocked, + poll_until_guardrail_applied, ) from lifecycle import ResourceManager from models import ChatResponse, ChatTool, ChatToolFunction @@ -84,8 +85,8 @@ def _register_tool_permission(client: GuardrailsClient, resources: ResourceManag resources.defer(lambda: client.delete_guardrail(guardrail_id)) -def _applied_guardrails(outcome: StreamingResponse) -> str: - return outcome.headers.get("x-litellm-applied-guardrails", "") +def _applied_guardrails(outcome: StreamingResponse) -> tuple[str, ...]: + return tuple(name.strip() for name in outcome.headers.get("x-litellm-applied-guardrails", "").split(",")) def _tool_call_names(response: ChatResponse) -> tuple[str, ...]: @@ -144,14 +145,17 @@ class TestToolPermissionPreCall: name = f"e2e-toolperm-allow-{unique_marker()}" _register_tool_permission(client, resources, name=name) - outcome = client.chat_raw( - scoped_key, - MODEL, - TOOL_PROMPT, - guardrails=[name], - max_tokens=128, - tools=[ALLOWED_TOOL], - tool_choice="required", + outcome = poll_until_guardrail_applied( + lambda: client.chat_raw( + scoped_key, + MODEL, + TOOL_PROMPT, + guardrails=[name], + max_tokens=128, + tools=[ALLOWED_TOOL], + tool_choice="required", + ), + name, ) assert outcome.ok, f"the permitted tool must be served, got {outcome.status_code}: {outcome.body[:400]}" From 9cb5d9b76c0c6b6e7a20e852e9673c9ea410c3f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:55:19 -0700 Subject: [PATCH 137/411] fix(proxy): gate streaming pipelines on a route-resolved guardrail translation The per-chunk hook skipped pipeline-managed guardrails whenever the request route was empty, while the gated stream could still fail to resolve a translation and release the buffered stream ungoverned. The gate now needs a translation resolved from the route, the iterator hook resolves it once and hands it to the gated stream, and the ungoverned release branch is gone. --- litellm/proxy/utils.py | 27 ++++------ .../proxy_logging/test_guardrail_pipeline.py | 53 +++++++++++++++---- 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c38eb900a05..37d077d607c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -192,6 +192,7 @@ if TYPE_CHECKING: from prisma.types import HttpConfig from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.models.team import LiteLLM_TeamTableCachedObj from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction @@ -559,7 +560,7 @@ def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> def _route_supports_streaming_pipelines(user_api_key_dict: UserAPIKeyAuth) -> bool: - return not user_api_key_dict.request_route or resolve_endpoint_translation(user_api_key_dict, None) is not None + return resolve_endpoint_translation(user_api_key_dict, None) is not None def _stream_gated_guardrail_names( @@ -3435,12 +3436,16 @@ class ProxyLogging: ), ) - if post_call_pipelines: + pipeline_translation: Final = ( + resolve_endpoint_translation(user_api_key_dict, None) if post_call_pipelines else None + ) + if pipeline_translation is not None: current_response = self._pipeline_gated_stream( response=current_response, user_api_key_dict=user_api_key_dict, request_data=request_data, pipelines=post_call_pipelines, + translation=pipeline_translation, ) try: @@ -3464,6 +3469,7 @@ class ProxyLogging: user_api_key_dict: UserAPIKeyAuth, request_data: dict, # mutable-ok: same request-payload shape the hooks mutate pipelines: "tuple[tuple[str, GuardrailPipeline], ...]", + translation: "tuple[str, BaseTranslation]", ) -> "AsyncGenerator[Any, None]": """ Execute post_call policy pipelines against a streamed response. @@ -3478,9 +3484,8 @@ class ProxyLogging: rewritten chunks, so rewrites chain). A rewrite the translation cannot deliver yet (one on a route without write-back, or a shape the route refuses) is discarded by the executor and the original chunks are - released, as is a buffered shape no translation resolves; a block or - modify_response terminates with the translation's block chunks or the - raised error. + released; a block or modify_response terminates with the translation's + block chunks or the raised error. """ buffered: Final[list[object]] = [] # mutable-ok: accumulates the stream before the pipeline verdict async for item in response: @@ -3488,17 +3493,7 @@ class ProxyLogging: if not buffered: return - resolved: Final = resolve_endpoint_translation(user_api_key_dict, buffered[0]) - if resolved is None: - verbose_proxy_logger.warning( - "Policies with post_call guardrail pipelines cannot scan this streaming response shape yet; " - "the stream is released ungoverned by them: %s", - ", ".join(policy_name for policy_name, _pipeline in pipelines), - ) - for buffered_item in buffered: - yield buffered_item - return - call_type, endpoint_translation = resolved + call_type, endpoint_translation = translation for policy_name, pipeline in pipelines: result: PipelineExecutionResult = await PipelineExecutor.execute_steps( diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index bf71780ad7b..3b59bbac3fd 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -1969,28 +1969,61 @@ async def test_streaming_iterator_hook_pipeline_releases_stream_echoed_in_anothe @pytest.mark.asyncio -async def test_streaming_iterator_hook_pipeline_releases_originals_on_unresolvable_response_shape( +async def test_streaming_iterator_hook_skips_pipeline_and_warns_without_request_route( proxy_logging, make_user_api_key_auth, monkeypatch, caplog ): seen: Dict[str, Any] = {} monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)]) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _post_call_pipeline_data(stream=True) - chunks = [object(), object()] - delivered: List[Any] = [] + chunks = _stream_chunks() with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - async for item in proxy_logging.async_post_call_streaming_iterator_hook( - user_api_key_dict=make_user_api_key_auth(), - response=_async_chunk_iter(chunks), - request_data=data, - ): - delivered.append(item) + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(), + response=_async_chunk_iter(chunks), + request_data=data, + ) + ] assert [item is chunk for item, chunk in zip(delivered, chunks)] == [True, True] assert len(delivered) == 2 assert seen.get("count") is None - assert any("response-governance" in message and "shape" in message for message in _warnings(caplog)) + assert any("response-governance" in message and "route None" in message for message in _warnings(caplog)) + + +@pytest.mark.asyncio +async def test_per_chunk_streaming_hook_runs_pipeline_managed_guardrail_without_request_route( + proxy_logging, make_user_api_key_auth, monkeypatch +): + seen: Dict[str, Any] = {} + + class UnifiedRecordingGuardrail(CustomGuardrail): + async def async_post_call_streaming_hook(self, user_api_key_dict, response): + seen[self.guardrail_name] = seen.get(self.guardrail_name, 0) + 1 + return None + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return inputs + + monkeypatch.setattr( + litellm, + "callbacks", + [UnifiedRecordingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True)], + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + result = await proxy_logging.async_post_call_streaming_hook( + data=data, + response=_stream_chunks()[0], + user_api_key_dict=make_user_api_key_auth(), + ) + + assert result is not None + assert seen["gr-post"] == 1 def _anthropic_sse_chunks() -> List[bytes]: From 0f224dd4ed1fb4bed9ff67d8d8a095e0d032fcae Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:59:34 -0700 Subject: [PATCH 138/411] fix: sign Bedrock guardrail, embedding, and Mantle requests off the event loop Bedrock Guardrails resolved credentials and signed inline on the loop in its three async paths, Titan embeddings signed each batch item inline in the async loop, and Bedrock Mantle requests slipped past the off-loop gate because BedrockMantleAuthMixin composes a BaseAWSLLM instead of inheriting from it. Introduce the SignsRequestsWithAWS marker that both BaseAWSLLM and the Mantle mixin carry so sign_request_off_loop_if_aws covers Mantle, and move the guardrail and embedding signing into asyncio.to_thread. Every new test fails at the previous tip. --- litellm/llms/bedrock/base_aws_llm.py | 8 +++- litellm/llms/bedrock/embed/embedding.py | 3 +- litellm/llms/bedrock_mantle/common_utils.py | 4 +- .../guardrail_hooks/bedrock_guardrails.py | 14 +++++-- .../test_bedrock_async_invoke_embedding.py | 35 ++++++++++++++++ .../bedrock/embed/test_bedrock_embedding.py | 42 +++++++++++++++++++ .../test_bedrock_mantle_transformation.py | 28 +++++++++++++ .../test_bedrock_guardrails.py | 36 ++++++++++++++++ .../test_bedrock_invoke_guardrail_checks.py | 33 +++++++++++++++ 9 files changed, 194 insertions(+), 9 deletions(-) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 1c7268ffa22..eddd4d6f826 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -81,7 +81,11 @@ class AwsAuthError(Exception): super().__init__(self.message) # Call the base class constructor with the parameters it needs -class BaseAWSLLM: +class SignsRequestsWithAWS: + pass + + +class BaseAWSLLM(SignsRequestsWithAWS): # Process-wide IAM credential cache (shared across instances — Bedrock passthrough is per-request). # Storage is in-process memory only: no Redis backend unless attached elsewhere. Entry TTL: static # access-key + secret + region use ``_get_default_ttl_for_boto3_credentials`` (~59 minutes); ambient @@ -1701,6 +1705,6 @@ async def sign_request_off_loop_if_aws( *args: _SignParams.args, **kwargs: _SignParams.kwargs, # kwargs-ok: ParamSpec forwarding keeps the wrapped sign_request signature ) -> _SignedRequest: - if isinstance(provider_config, BaseAWSLLM): + if isinstance(provider_config, SignsRequestsWithAWS): return await asyncio.to_thread(sign_request, *args, **kwargs) return sign_request(*args, **kwargs) diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index d0e8d734c5e..2c66fa1ef06 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -357,7 +357,8 @@ class BedrockEmbedding(BaseAWSLLM): if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} - prepped = self.get_request_headers( + prepped = await asyncio.to_thread( + self.get_request_headers, credentials=credentials, aws_region_name=aws_region_name, extra_headers=extra_headers, diff --git a/litellm/llms/bedrock_mantle/common_utils.py b/litellm/llms/bedrock_mantle/common_utils.py index 850738bc320..ac94d9cb922 100644 --- a/litellm/llms/bedrock_mantle/common_utils.py +++ b/litellm/llms/bedrock_mantle/common_utils.py @@ -23,7 +23,7 @@ from botocore.exceptions import ( ProfileNotFound, ) -from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, SignsRequestsWithAWS from litellm.secret_managers.main import get_secret_str BEDROCK_MANTLE_DEFAULT_REGION: Final = "us-east-1" @@ -55,7 +55,7 @@ def resolve_mantle_region(params: Mapping[str, object]) -> str: ) -class BedrockMantleAuthMixin: +class BedrockMantleAuthMixin(SignsRequestsWithAWS): _aws_signer: BaseAWSLLM @staticmethod diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 7204839f6d3..0ae2c48a3b4 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -917,7 +917,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): source, ) return BedrockGuardrailResponse() - credentials, aws_region_name = self._load_credentials(bearer_token=bedrock_bearer_token(api_key)) + credentials, aws_region_name = await asyncio.to_thread( + self._load_credentials, bearer_token=bedrock_bearer_token(api_key) + ) allow_chunking: Final = not self._content_uses_contextual_grounding(content) completed_chunk_usages: Final[list[BedrockGuardrailUsage]] = [] # mutable-ok: billed-chunk usage accumulator @@ -1178,7 +1180,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): **base_request_data, "content": content, } # mutable-ok: outbound JSON request body - prepared_request: Final = self._prepare_request( + prepared_request: Final = await asyncio.to_thread( + self._prepare_request, credentials=credentials, data=bedrock_request_data, optional_params=self.optional_params, @@ -1875,10 +1878,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return BedrockGuardrailResponse() api_key: Final[str | None] = request_data.get("api_key") if request_data else None - credentials, aws_region_name = self._load_credentials(bearer_token=bedrock_bearer_token(api_key)) + credentials, aws_region_name = await asyncio.to_thread( + self._load_credentials, bearer_token=bedrock_bearer_token(api_key) + ) body: Final[dict[str, object]] = {"messages": checks_messages, "checks": self.checks} - prepared_request: Final = self._prepare_request( + prepared_request: Final = await asyncio.to_thread( + self._prepare_request, credentials=credentials, data=body, optional_params=self.optional_params, diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py index 74a55cc1ef2..d8dfaaeac99 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py @@ -1,11 +1,15 @@ import json +import asyncio from unittest.mock import Mock, patch +import httpx import pytest +import respx import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.base import HiddenParams +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe # Mock async invoke responses async_invoke_response = { @@ -383,3 +387,34 @@ class TestBedrockAsyncInvokeEmbedding: async_endpoint == "https://bedrock-runtime.us-east-1.amazonaws.com/async-invoke" ) + + +@pytest.mark.asyncio +async def test_async_invoke_status_signs_off_the_event_loop(monkeypatch): + """Regression for issue #40165: the GetAsyncInvoke poll is a signed GET, and botocore refreshes + expiring credentials inside that signing with a blocking HTTP call, so it must run on a worker + thread to keep the loop serving other requests.""" + from litellm.llms.bedrock.embed.embedding import BedrockEmbedding + + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + embedder = BedrockEmbedding() + probe = EventLoopProbe() + + with ( + patch.object(embedder, "_load_credentials", return_value=(probe.credentials(), "us-east-1")), + respx.mock, + ): + route = respx.get(url__regex=r"https://bedrock-runtime\.us-east-1\.amazonaws\.com/async-invoke/.*").mock( + return_value=httpx.Response(200, json=async_invoke_status_response) + ) + release = asyncio.create_task(probe.release_refresh_from_the_loop()) + status = await embedder._get_async_invoke_status( + invocation_arn=async_invoke_status_response["invocationArn"], aws_region_name="us-east-1" + ) + await release + + assert status["status"] == "InProgress" + assert "Authorization" in route.calls.last.request.headers + assert probe.served_during_refresh is True diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py index 50f8bbcf584..4ebf08dc709 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py @@ -1,11 +1,16 @@ import json +import asyncio import os from unittest.mock import Mock, patch +from unittest.mock import AsyncMock, MagicMock import pytest +import httpx import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.bedrock.embed.embedding import BedrockEmbedding +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe # Mock responses for different embedding models titan_embedding_response = {"embedding": [0.1, 0.2, 0.3], "inputTextTokenCount": 10} @@ -1059,3 +1064,40 @@ def test_bedrock_embedding_bearer_token_never_runs_the_sigv4_credential_chain(mo assert response.data[0]["embedding"] == titan_embedding_response["embedding"] assert mock_post.call_args.kwargs["headers"]["Authorization"] == "Bearer env-bearer-token-12345" + + +@pytest.mark.asyncio +async def test_async_single_func_embeddings_signs_off_the_event_loop(monkeypatch): + """Regression for issue #40165: Titan, Nova, and TwelveLabs embeddings sign one SigV4 request per + input, and botocore refreshes expiring credentials inside that signing with a blocking HTTP call, + so each signing must run on a worker thread to keep the loop serving other requests.""" + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + probe = EventLoopProbe() + client = MagicMock() + client.__class__ = AsyncHTTPHandler + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json=titan_embedding_response, + request=httpx.Request("POST", "https://bedrock-runtime.us-west-2.amazonaws.com"), + ) + ) + + release = asyncio.create_task(probe.release_refresh_from_the_loop()) + response = await BedrockEmbedding()._async_single_func_embeddings( + client=client, + timeout=None, + batch_data=[{"inputText": test_input}], + credentials=probe.credentials(), + extra_headers=None, + endpoint_url="https://bedrock-runtime.us-west-2.amazonaws.com/model/amazon.titan-embed-text-v1/invoke", + aws_region_name="us-west-2", + model="amazon.titan-embed-text-v1", + logging_obj=MagicMock(), + provider="amazon", + ) + await release + + assert response.data[0]["embedding"] == titan_embedding_response["embedding"] + assert "Authorization" in client.post.call_args.kwargs["headers"] + assert probe.served_during_refresh is True diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 1be94d4daa2..e83a844c87e 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -6,15 +6,20 @@ API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.ht """ import json +import asyncio from unittest.mock import patch import httpx import pytest +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest import litellm from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig +from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws from litellm.types.utils import LlmProviders +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe @pytest.fixture @@ -710,3 +715,26 @@ def test_gemma_4_models_register_under_bedrock_mantle(local_cost_map, model_id): resolved_model, provider, _, _ = litellm.get_llm_provider(full_model_name) assert provider == "bedrock_mantle" assert resolved_model == model_id + + +@pytest.mark.asyncio +async def test_mantle_signing_runs_off_the_event_loop(): + """Regression for issue #40165: Mantle signs with SigV4 through a composed BaseAWSLLM, so the + off-loop gate must recognise it too, or its credential refresh blocks the loop like Bedrock's did.""" + probe = EventLoopProbe() + + def sign(headers: dict[str, str]) -> dict[str, str]: + request = AWSRequest( + method="POST", url="https://bedrock-mantle.us-east-1.api.aws/v1/responses", data="{}", headers=headers + ) + SigV4Auth(probe.credentials(), "bedrock", "us-east-1").add_auth(request) + return dict(request.headers) + + release = asyncio.create_task(probe.release_refresh_from_the_loop()) + signed = await sign_request_off_loop_if_aws( + BedrockMantleChatConfig(), sign, headers={"Content-Type": "application/json"} + ) + await release + + assert "Authorization" in signed + assert probe.served_during_refresh is True diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 7da55f22bda..c0762edec92 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -3,6 +3,8 @@ Unit tests for Bedrock Guardrails """ import json +import asyncio +from datetime import datetime, timezone import sys from unittest.mock import AsyncMock, MagicMock, patch @@ -28,6 +30,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockTextContent, ) from litellm.types.utils import CallTypes, ModelResponse +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe @pytest.mark.asyncio @@ -5842,3 +5845,36 @@ async def test_bearer_token_never_runs_the_sigv4_credential_chain(monkeypatch): assert response["action"] == "NONE" assert mock_post.call_args.kwargs["headers"]["Authorization"] == "Bearer env-bearer-token-12345" + + +@pytest.mark.asyncio +async def test_apply_guardrail_signs_off_the_event_loop(monkeypatch): + """Regression for issue #40165: the ApplyGuardrail request is signed with SigV4, and botocore + refreshes expiring credentials inside that signing with a blocking HTTP call, so it must run + on a worker thread to keep the loop serving other requests.""" + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + guardrail = BedrockGuardrail(guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT") + probe = EventLoopProbe() + allowed = httpx.Response( + 200, + json={"action": "NONE", "outputs": [], "assessments": []}, + request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com"), + ) + + with patch.object(guardrail.async_handler, "post", new=AsyncMock(return_value=allowed)): + release = asyncio.create_task(probe.release_refresh_from_the_loop()) + response = await guardrail._post_apply_guardrail_content( + content=[{"text": {"text": "hello"}}], + base_request_data={"source": "INPUT"}, + credentials=probe.credentials(), + aws_region_name="us-east-1", + api_key=None, + request_data={}, + event_type=GuardrailEventHooks.pre_call, + start_time=datetime.now(timezone.utc), + completed_chunk_usages=[], + ) + await release + + assert response["action"] == "NONE" + assert probe.served_during_refresh is True diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py index f4af77d2e40..bbc8fd539a3 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py @@ -5,10 +5,12 @@ All Bedrock HTTP calls are mocked; no real AWS calls are made. """ import json +import asyncio import logging from unittest.mock import AsyncMock, MagicMock, patch import pytest +import httpx from fastapi import HTTPException @@ -21,6 +23,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockGuardrailResponse, ) from litellm.types.utils import Choices, Message, ModelResponse +from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe CONTENT_FILTER_CHECKS = {"contentFilter": {"categories": [{"category": "VIOLENCE"}]}} @@ -861,3 +864,33 @@ async def test_checks_bearer_token_never_runs_the_sigv4_credential_chain(monkeyp {"check": "contentFilter", "category": "VIOLENCE", "severityScore": 0.8} ] assert post.call_args.kwargs["headers"]["Authorization"] == "Bearer env-bearer-token-12345" + + +@pytest.mark.asyncio +async def test_invoke_guardrail_checks_signs_off_the_event_loop(monkeypatch): + """Regression for issue #40165: the checks request is signed with SigV4, and botocore refreshes + expiring credentials inside that signing with a blocking HTTP call, so it must run on a worker + thread to keep the loop serving other requests.""" + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + g = BedrockGuardrail(checks=CONTENT_FILTER_CHECKS, content_filter_threshold=0.5) + probe = EventLoopProbe() + allowed = httpx.Response( + 200, + json={"results": {"contentFilter": {"results": [{"category": "VIOLENCE", "severityScore": 0.1}]}}}, + request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com"), + ) + + with ( + patch.object(g, "_load_credentials", return_value=(probe.credentials(), "us-east-1")), + patch.object(g.async_handler, "post", new=AsyncMock(return_value=allowed)), + ): + release = asyncio.create_task(probe.release_refresh_from_the_loop()) + response = await g.make_bedrock_api_request( + source="INPUT", + messages=[{"role": "user", "content": "hello"}], + request_data={"messages": []}, + ) + await release + + assert response == BedrockGuardrailResponse() + assert probe.served_during_refresh is True From 88d2d775534457b3397c9332ca968b7a8707a161 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:59:55 -0700 Subject: [PATCH 139/411] feat(hosted_vllm): add image edit support Register a HostedVLLMImageEditConfig so hosted_vllm/ deployments route POST /v1/images/edits to the vLLM-Omni OpenAI-compatible endpoint instead of failing with 'image edit is not supported for hosted_vllm' before any request is sent --- .../llms/hosted_vllm/image_edit/__init__.py | 9 ++ .../hosted_vllm/image_edit/transformation.py | 42 +++++++ litellm/utils.py | 4 + ...t_hosted_vllm_image_edit_transformation.py | 114 ++++++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 litellm/llms/hosted_vllm/image_edit/__init__.py create mode 100644 litellm/llms/hosted_vllm/image_edit/transformation.py create mode 100644 tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py diff --git a/litellm/llms/hosted_vllm/image_edit/__init__.py b/litellm/llms/hosted_vllm/image_edit/__init__.py new file mode 100644 index 00000000000..27e005e8a0d --- /dev/null +++ b/litellm/llms/hosted_vllm/image_edit/__init__.py @@ -0,0 +1,9 @@ +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig + +from .transformation import HostedVLLMImageEditConfig + +__all__ = ("HostedVLLMImageEditConfig",) + + +def get_hosted_vllm_image_edit_config(model: str) -> BaseImageEditConfig: + return HostedVLLMImageEditConfig() diff --git a/litellm/llms/hosted_vllm/image_edit/transformation.py b/litellm/llms/hosted_vllm/image_edit/transformation.py new file mode 100644 index 00000000000..bd27c962da8 --- /dev/null +++ b/litellm/llms/hosted_vllm/image_edit/transformation.py @@ -0,0 +1,42 @@ +"""Image edits for Hosted VLLM (vLLM-Omni OpenAI-compatible /v1/images/edits).""" + +from typing import Final + +from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig +from litellm.secret_managers.main import get_secret_str + + +class HostedVLLMImageEditConfig(OpenAIImageEditConfig): + """ + vLLM-Omni images edits API follows the OpenAI multipart contract. + + https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/images_api/ + """ + + def validate_environment( + self, + headers: dict, # mutable-ok: BaseImageEditConfig contract + model: str, + api_key: str | None = None, + litellm_params: dict | None = None, # mutable-ok: BaseImageEditConfig contract + api_base: str | None = None, + ) -> dict: # mutable-ok: BaseImageEditConfig contract + resolved_key: Final = api_key or get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key" + return {**headers, "Authorization": f"Bearer {resolved_key}"} # mutable-ok: httpx headers are a dict + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, # mutable-ok: BaseImageEditConfig contract + ) -> str: + resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") + if resolved_api_base is None: + raise ValueError( + "api_base not set for Hosted VLLM images edits API. " + "Set via api_base parameter or HOSTED_VLLM_API_BASE environment variable" + ) + trimmed: Final = resolved_api_base.rstrip("/") + if trimmed.endswith("/v1"): + return f"{trimmed}/images/edits" + return f"{trimmed}/v1/images/edits" diff --git a/litellm/utils.py b/litellm/utils.py index 141ec323776..0f2b2f9042d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9239,6 +9239,10 @@ class ProviderConfigManager: from litellm.llms.openai.image_edit import get_openai_image_edit_config return get_openai_image_edit_config(model=model) + if LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config + + return get_hosted_vllm_image_edit_config(model=model) elif LlmProviders.AZURE == provider: from litellm.llms.azure.image_edit.transformation import ( AzureImageEditConfig, diff --git a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py new file mode 100644 index 00000000000..e1167b92553 --- /dev/null +++ b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py @@ -0,0 +1,114 @@ +"""Tests for hosted_vllm image edits (vLLM-Omni /v1/images/edits).""" + +import httpx +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config +from litellm.llms.hosted_vllm.image_edit.transformation import HostedVLLMImageEditConfig +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +PNG_BYTES = b"\x89PNG\r\n\x1a\nfakepng" +MODEL = "Qwen/Qwen-Image-Edit-2511" + + +@pytest.fixture(autouse=True) +def _clear_hosted_vllm_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + + +def test_provider_config_registration(): + config = ProviderConfigManager.get_provider_image_edit_config( + model=f"hosted_vllm/{MODEL}", + provider=LlmProviders.HOSTED_VLLM, + ) + + assert isinstance(config, HostedVLLMImageEditConfig) + assert isinstance(get_hosted_vllm_image_edit_config(MODEL), HostedVLLMImageEditConfig) + + +@pytest.mark.parametrize( + "api_base", + ["http://localhost:8091", "http://localhost:8091/", "http://localhost:8091/v1", "http://localhost:8091/v1/"], +) +def test_get_complete_url_appends_images_edits(api_base: str): + config = HostedVLLMImageEditConfig() + + assert ( + config.get_complete_url(model=MODEL, api_base=api_base, litellm_params={}) + == "http://localhost:8091/v1/images/edits" + ) + + +def test_get_complete_url_falls_back_to_env(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://vllm-omni:8000/v1") + config = HostedVLLMImageEditConfig() + + assert ( + config.get_complete_url(model=MODEL, api_base=None, litellm_params={}) + == "http://vllm-omni:8000/v1/images/edits" + ) + + +def test_get_complete_url_requires_api_base(): + config = HostedVLLMImageEditConfig() + + with pytest.raises(ValueError, match="api_base not set"): + config.get_complete_url(model=MODEL, api_base=None, litellm_params={}) + + +def test_validate_environment_defaults_to_fake_api_key(): + headers = HostedVLLMImageEditConfig().validate_environment(headers={}, model=MODEL) + + assert headers == {"Authorization": "Bearer fake-api-key"} + + +def test_validate_environment_uses_provided_api_key_and_keeps_headers(): + headers = HostedVLLMImageEditConfig().validate_environment( + headers={"X-Test": "1"}, + model=MODEL, + api_key="my-custom-key", + ) + + assert headers == {"X-Test": "1", "Authorization": "Bearer my-custom-key"} + + +def test_validate_environment_falls_back_to_env_api_key(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "env-key") + + headers = HostedVLLMImageEditConfig().validate_environment(headers={}, model=MODEL) + + assert headers["Authorization"] == "Bearer env-key" + + +def test_image_edit_posts_multipart_to_vllm_omni(): + captured: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response(200, json={"created": 1712697600, "data": [{"b64_json": "aW1n"}]}) + + response = litellm.image_edit( + model=f"hosted_vllm/{MODEL}", + image=PNG_BYTES, + prompt="add a hat", + api_base="http://localhost:8091", + api_key="test-key", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))), + seed=42, + ) + + assert response.data + assert len(captured) == 1 + request = captured[0] + assert str(request.url) == "http://localhost:8091/v1/images/edits" + assert request.headers["authorization"] == "Bearer test-key" + assert request.headers["content-type"].startswith("multipart/form-data") + assert b'name="image[]"' in request.content + assert PNG_BYTES in request.content + assert f'name="model"\r\n\r\n{MODEL}'.encode() in request.content + assert b'name="prompt"\r\n\r\nadd a hat' in request.content + assert b'name="seed"\r\n\r\n42' in request.content From b8be30219c5e5dea4343b3a924b89494e7350303 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 17:02:00 -0700 Subject: [PATCH 140/411] test: stop guardrail retries at the polling deadline --- tests/e2e/guardrails/guardrails_client.py | 7 +++++-- tests/e2e/guardrails/test_guardrails_client.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 3770de0c6d5..ed112a79b9b 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -415,13 +415,16 @@ def poll_until_guardrail_applied( sleep: Callable[[float], None] = time.sleep, ) -> StreamingResponse: deadline: Final = now() + timeout + if not (result := call()).ok: + return result while ( - (result := call()).ok - and guardrail_name + guardrail_name not in (name.strip() for name in result.headers.get("x-litellm-applied-guardrails", "").split(",")) and (remaining := deadline - now()) > 0 ): sleep(min(interval, remaining)) + if now() >= deadline or not (result := call()).ok: + break return result diff --git a/tests/e2e/guardrails/test_guardrails_client.py b/tests/e2e/guardrails/test_guardrails_client.py index aab163d86d2..423c2ede599 100644 --- a/tests/e2e/guardrails/test_guardrails_client.py +++ b/tests/e2e/guardrails/test_guardrails_client.py @@ -40,13 +40,16 @@ def test_waits_for_requested_guardrail_after_an_unrelated_global_guardrail() -> def test_missing_exact_guardrail_returns_failure_evidence_at_deadline(applied: str) -> None: clock: Final = Clock() missing: Final = _response(applied) + responses: Final = iter((missing, missing, missing)) result: Final = poll_until_guardrail_applied( - lambda: missing, "tool-permission", timeout=5, interval=2, now=clock.now, sleep=clock.sleep + lambda: next(responses), "tool-permission", timeout=5, interval=2, now=clock.now, sleep=clock.sleep ) assert result is missing assert clock.elapsed == 5 + with pytest.raises(StopIteration): + next(responses) @pytest.mark.parametrize("status", (400, 401, 429, 500)) From 0c58346ba925d0a15ebcb38b162b9224446a50d6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:07:37 -0700 Subject: [PATCH 141/411] fix(proxy): warn when a deferred background policy was matched through a request tag Retrieval re-matches only the key, team, and model scopes, so a post_call policy that reached a pending background response through a request-body tag does not govern the completed response. Log that at submit, next to the deferral, and cover the retrieval re-match with tag-scoped tests. --- litellm/proxy/utils.py | 22 +++++++++ .../policy_engine/test_response_retrieval.py | 22 +++++++++ .../proxy_logging/test_guardrail_pipeline.py | 47 ++++++++++++++++++- 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 4b62fc4cdae..d144aa113d6 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -613,9 +613,31 @@ def _defer_post_call_pipelines( response.status, ", ".join(policy_name for policy_name, _pipeline in deferred), ) + tag_matched: Final = _tag_matched_deferrals(data, deferred) + if tag_matched: + verbose_proxy_logger.warning( + "Policy engine: background response %s matched post_call policies through a request tag at submit; " + "retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body " + "does not govern the completed response: %s", + response.id, + ", ".join(f"{policy_name} ({source})" for policy_name, source in tag_matched), + ) _withdraw_deferred_claims(data, deferred) +def _tag_matched_deferrals( + data: Mapping[str, object], deferred: Sequence[tuple[str, "GuardrailPipeline"]] +) -> tuple[tuple[str, str], ...]: + sources: Final = _policy_state_metadata(data).get("policy_sources") + if not isinstance(sources, dict): + return () + return tuple( + (policy_name, str(sources[policy_name])) + for policy_name, _pipeline in deferred + if policy_name in sources and "tag:" in str(sources[policy_name]) + ) + + def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: unsupported: Final = tuple( dict.fromkeys( diff --git a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py index 9dcbef888ce..1fa1161191d 100644 --- a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py +++ b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py @@ -68,6 +68,7 @@ def policy_engine(): "response-governance": _pipeline_policy("output-word-filter"), "input-governance": _pipeline_policy("input-word-filter", mode="pre_call"), "team-governance": _pipeline_policy("team-word-filter"), + "tag-governance": _pipeline_policy("tag-word-filter"), } ) attachment_registry.load_attachments( @@ -75,6 +76,7 @@ def policy_engine(): {"policy": "response-governance", "models": [GOVERNED_MODEL_GROUP]}, {"policy": "input-governance", "models": [GOVERNED_MODEL_GROUP]}, {"policy": "team-governance", "teams": ["governed-team"]}, + {"policy": "tag-governance", "tags": ["governed"]}, ] ) yield @@ -119,6 +121,26 @@ def test_key_and_team_context_also_governs_retrieval(policy_engine): assert _attached_pipelines(data) == (("team-governance", "team-word-filter"),) +def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag(policy_engine): + data = _retrieval_data(GOVERNED_MODEL_ID) + + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + + assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) + + +def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag(policy_engine): + data: dict[str, object] = { + "response_id": _encoded_response_id(UNGOVERNED_MODEL_ID), + "litellm_metadata": {"tags": ["governed"]}, + } + + attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) + + assert _attached_pipelines(data) == (("tag-governance", "tag-word-filter"),) + assert data["litellm_metadata"]["policy_sources"] == {"tag-governance": "tag:governed"} + + def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine): data = _retrieval_data(UNGOVERNED_MODEL_ID) diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index d6a74d3bbd9..63afce5d968 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -1489,7 +1489,9 @@ def _output_passing_callbacks() -> list[CustomGuardrail]: return [OutputPassingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)] -def _claimed_post_call_pipeline_data(*policy_names: str, extra_guardrails: dict[str, list[str]] | None = None): +def _claimed_post_call_pipeline_data( + *policy_names: str, extra_guardrails: dict[str, list[str]] | None = None, policy_source: str = "model:m" +): from litellm.proxy.policy_engine.policy_registry import get_policy_registry step = {"guardrail": "gr-post", "on_pass": "allow", "on_fail": "block"} @@ -1511,7 +1513,7 @@ def _claimed_post_call_pipeline_data(*policy_names: str, extra_guardrails: dict[ "_pipeline_managed_guardrails": {"gr-post"}, "applied_policies": list(policy_names), "applied_guardrails": ["gr-post", *(g for gs in (extra_guardrails or {}).values() for g in gs)], - "policy_sources": {policy_name: "model:m" for policy_name in policy_names}, + "policy_sources": {policy_name: policy_source for policy_name in policy_names}, }, } @@ -1542,6 +1544,47 @@ async def test_pending_background_response_withdraws_the_deferred_policy_claims( assert "applied_guardrails" not in data["metadata"] +@pytest.mark.asyncio +async def test_pending_background_response_warns_when_the_deferred_policy_was_matched_through_a_tag( + proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry, caplog +): + monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _claimed_post_call_pipeline_data("response-governance", policy_source="tag:governed+model:m") + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + out = await proxy_logging.post_call_success_hook( + data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth() + ) + + assert out.status == "queued" + assert "policy_sources" not in data["metadata"] + assert [ + message for message in _warnings(caplog) if "response-governance (tag:governed+model:m)" in message + ] == [ + "Policy engine: background response resp_bg matched post_call policies through a request tag at submit; " + "retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body " + "does not govern the completed response: response-governance (tag:governed+model:m)" + ] + + +@pytest.mark.asyncio +async def test_pending_background_response_matched_through_its_model_does_not_warn( + proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry, caplog +): + monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await proxy_logging.post_call_success_hook( + data=_claimed_post_call_pipeline_data("response-governance"), + response=_background_response("queued"), + user_api_key_dict=make_user_api_key_auth(), + ) + + assert _warnings(caplog) == [] + + @pytest.mark.asyncio async def test_pending_background_response_keeps_the_claim_of_a_policy_that_runs_outside_its_pipeline( proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry From 831a2a13fbd87aee9e77cc6831963ac4a9975fbb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:12:18 -0700 Subject: [PATCH 142/411] fix(azure): price azure_ai transcriptions at the azure_ai cost-map entry --- litellm/llms/azure/audio_transcriptions.py | 7 ++- litellm/main.py | 1 + .../llms/azure/test_audio_transcriptions.py | 61 +++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/llms/azure/test_audio_transcriptions.py diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 4a5ed2ccb0c..564ec94ba6b 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -37,6 +37,7 @@ class AzureAudioTranscription(AzureChatCompletion): azure_ad_token: str | None = None, atranscription: bool = False, litellm_params: dict | None = None, + custom_llm_provider: str = "azure", ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} @@ -53,6 +54,7 @@ class AzureAudioTranscription(AzureChatCompletion): logging_obj=logging_obj, model=model, litellm_params=litellm_params, + custom_llm_provider=custom_llm_provider, ) azure_client: Final = self.get_azure_openai_client( @@ -99,7 +101,7 @@ class AzureAudioTranscription(AzureChatCompletion): additional_args={"complete_input_dict": data}, original_response=stringified_response, ) - hidden_params: Final = {"model": model, "custom_llm_provider": "azure"} + hidden_params: Final = {"model": model, "custom_llm_provider": custom_llm_provider} final_response: Final[TranscriptionResponse] = convert_to_model_response_object( response_object=stringified_response, model_response_object=model_response, @@ -122,6 +124,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=None, max_retries=None, litellm_params: dict | None = None, + custom_llm_provider: str = "azure", ) -> TranscriptionResponse: response = None try: @@ -178,7 +181,7 @@ class AzureAudioTranscription(AzureChatCompletion): }, original_response=stringified_response, ) - hidden_params: Final = {"model": model, "custom_llm_provider": "azure"} + hidden_params: Final = {"model": model, "custom_llm_provider": custom_llm_provider} response = convert_to_model_response_object( _response_headers=headers, response_object=stringified_response, diff --git a/litellm/main.py b/litellm/main.py index 75b7f7f10a5..11b30c240aa 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7805,6 +7805,7 @@ def transcription( azure_ad_token=azure_ad_token, max_retries=max_retries, litellm_params=litellm_params_dict, + custom_llm_provider=custom_llm_provider, ) elif custom_llm_provider == "openai" or (custom_llm_provider in litellm.openai_compatible_providers): api_base = ( diff --git a/tests/test_litellm/llms/azure/test_audio_transcriptions.py b/tests/test_litellm/llms/azure/test_audio_transcriptions.py new file mode 100644 index 00000000000..cd5fcbd85a9 --- /dev/null +++ b/tests/test_litellm/llms/azure/test_audio_transcriptions.py @@ -0,0 +1,61 @@ +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest +from openai import AzureOpenAI + +import litellm +from litellm.cost_calculator import completion_cost +from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration + +AUDIO_FILE: Final = Path(__file__).parents[3] / "gettysburg.wav" +WHISPER_COST_PER_SECOND: Final = 0.0001 + + +def _transcription_client() -> AzureOpenAI: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"text": "Four score and seven years ago"}) + + return AzureOpenAI( + api_key="test-key", + api_version="2024-06-01", + azure_endpoint="https://example.cognitiveservices.azure.com", + http_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + + +def test_azure_ai_transcription_is_priced_at_the_azure_ai_entry(): + with AUDIO_FILE.open("rb") as audio: + response = litellm.transcription( + model="azure_ai/whisper", + file=audio, + api_base="https://example.cognitiveservices.azure.com", + api_key="test-key", + api_version="2024-06-01", + client=_transcription_client(), + ) + with AUDIO_FILE.open("rb") as audio: + duration = calculate_request_duration(audio) + + assert duration is not None and duration > 0 + assert response._hidden_params["custom_llm_provider"] == "azure_ai" + assert completion_cost(completion_response=response, call_type="transcription") == pytest.approx( + WHISPER_COST_PER_SECOND * duration + ) + + +def test_azure_transcription_keeps_the_azure_provider(): + with AUDIO_FILE.open("rb") as audio: + response = litellm.transcription( + model="azure/whisper-1", + file=audio, + api_base="https://example.openai.azure.com", + api_key="test-key", + api_version="2024-06-01", + client=_transcription_client(), + ) + + assert response._hidden_params["custom_llm_provider"] == "azure" + assert json.loads(response.model_dump_json())["text"] == "Four score and seven years ago" From a892e67c40c3d84064f4c9f3d83c74bb1e4295a5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:05:06 -0700 Subject: [PATCH 143/411] fix(azure): match deployment segments case-insensitively and keep relay helpers immutable --- litellm/llms/azure/passthrough/transformation.py | 15 ++++++++++----- litellm/llms/openai/responses/transformation.py | 2 +- .../llm_passthrough_endpoints.py | 16 +++++++++------- .../test_azure_passthrough_transformation.py | 10 ++++++++++ 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 51a409c61da..69767bb29a3 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -1,5 +1,6 @@ import re from collections.abc import Callable, Collection, Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Optional import httpx @@ -55,9 +56,7 @@ def logged_responses_stream(all_chunks: Sequence[str], logging_obj: Logging) -> """A streaming logging object assembles the logged response from the terminal event, not from its body.""" from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig - terminal_event: Final = OpenAIResponsesAPIConfig.parse_terminal_event_from_stream_chunks( - all_chunks=list(all_chunks) - ) + terminal_event: Final = OpenAIResponsesAPIConfig.parse_terminal_event_from_stream_chunks(all_chunks=all_chunks) if terminal_event is None: return None logging_obj.call_type = ( @@ -83,7 +82,11 @@ def foreign_azure_deployment( if match is None: return None deployment: Final = match.group(1) - return None if deployment == model_group or deployment in served_models() else deployment + folded: Final = deployment.casefold() + if folded == model_group.casefold(): + return None + served: Final = frozenset(name.casefold() for name in served_models()) + return None if folded in served else deployment def without_api_version(api_base: str) -> str: @@ -119,7 +122,9 @@ class AzurePassthroughConfig(BasePassthroughConfig): relay_base: Final = without_api_version(base_target_url) if caller_api_version else base_target_url complete_url: Final = BaseAzureLLM._get_base_azure_url( api_base=relay_base, - litellm_params={**litellm_params, "api_version": caller_api_version or litellm_params.get("api_version")}, + litellm_params=MappingProxyType( + {**litellm_params, "api_version": caller_api_version or litellm_params.get("api_version")} + ), route=native_endpoint, ) return ( diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index cde399065fd..97e7bcc60b7 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -620,7 +620,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return event_pydantic_model.model_construct(**parsed_chunk) @staticmethod - def parse_terminal_event_from_stream_chunks(all_chunks: list[str]) -> ResponsesTerminalEvent | None: + def parse_terminal_event_from_stream_chunks(all_chunks: Sequence[str]) -> ResponsesTerminalEvent | None: for chunk_str in reversed(all_chunks): for event_model in (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent): try: diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 66987042962..086e1654f70 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -122,6 +122,10 @@ def is_passthrough_request_using_router_model(request_body: dict, llm_router: li return False +class RelayRejection(TypedDict): + error: ReadOnly[str] + + def _deployment_model_name(litellm_params: LiteLLMParamsTypedDict) -> str: model: Final = litellm_params.get("model", "") try: @@ -1543,13 +1547,11 @@ async def _relay_azure_router_model( endpoint, model, lambda: _models_served_by_group(llm_router, model) ) if foreign_deployment is not None: - raise HTTPException( - status_code=400, - detail={ - "error": f"deployment '{foreign_deployment}' in the path is not served by model group '{model}'; " - "put the model group name in the deployments segment" - }, - ) + rejection: Final[RelayRejection] = { + "error": f"deployment '{foreign_deployment}' in the path is not served by model group '{model}'; " + "put the model group name in the deployments segment" + } + raise HTTPException(status_code=400, detail=rejection) try: result: Final = await llm_router.allm_passthrough_route( model=model, diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 69fdd59a748..798ef1f5a4f 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -456,16 +456,26 @@ def test_azure_passthrough_is_streaming_request_reads_the_stream_flag(request_da ("gpt/openai/deployments/gpt/chat/completions", None), ("openai/deployments/gpt/chat/completions", None), ("gpt/openai/deployments/gpt-5.4-mini/chat/completions", None), + ("gpt/openai/deployments/GPT-5.4-MINI/chat/completions", None), + ("gpt/openai/deployments/Gpt/chat/completions", None), ("gpt/models/chat/completions", None), ("gpt/openai/deployments/gpt-5.4/chat/completions", "gpt-5.4"), ("gpt/openai/deployments/other-group/chat/completions", "other-group"), ("openai/deployments/victim/gpt/chat/completions", "victim"), + ("gpt/openai/deployments/GPT-5.4/chat/completions", "GPT-5.4"), ], ) def test_foreign_azure_deployment_names_a_segment_outside_the_group(endpoint, expected): assert foreign_azure_deployment(endpoint, "gpt", lambda: frozenset({"gpt-5.4-mini"})) == expected +def test_foreign_azure_deployment_skips_the_router_when_the_segment_is_the_group_itself(): + def served_models(): + raise AssertionError("the router must not be consulted for the group's own name") + + assert foreign_azure_deployment("gpt/openai/deployments/Gpt/chat/completions", "gpt", served_models) is None + + @pytest.mark.parametrize( "endpoint, expected", [ From 260097b2dde7f12f77784892e96e9956b0b19cc1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:16:34 -0700 Subject: [PATCH 144/411] refactor(hosted_vllm): drop redundant image edit docstrings --- litellm/llms/hosted_vllm/image_edit/transformation.py | 8 -------- .../test_hosted_vllm_image_edit_transformation.py | 2 -- 2 files changed, 10 deletions(-) diff --git a/litellm/llms/hosted_vllm/image_edit/transformation.py b/litellm/llms/hosted_vllm/image_edit/transformation.py index bd27c962da8..fa8c1dbfcfd 100644 --- a/litellm/llms/hosted_vllm/image_edit/transformation.py +++ b/litellm/llms/hosted_vllm/image_edit/transformation.py @@ -1,5 +1,3 @@ -"""Image edits for Hosted VLLM (vLLM-Omni OpenAI-compatible /v1/images/edits).""" - from typing import Final from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig @@ -7,12 +5,6 @@ from litellm.secret_managers.main import get_secret_str class HostedVLLMImageEditConfig(OpenAIImageEditConfig): - """ - vLLM-Omni images edits API follows the OpenAI multipart contract. - - https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/images_api/ - """ - def validate_environment( self, headers: dict, # mutable-ok: BaseImageEditConfig contract diff --git a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py index e1167b92553..5646f3d9d13 100644 --- a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py @@ -1,5 +1,3 @@ -"""Tests for hosted_vllm image edits (vLLM-Omni /v1/images/edits).""" - import httpx import pytest From 9e67c083092da5e20b85ed87e935d0af168cc66d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:16:39 -0700 Subject: [PATCH 145/411] fix(policy_engine): keep tool-only legacy streams deliverable A Messages stream that ends with only tool_use blocks reaches the legacy step without a texts key, while the non-streaming rescan of the same response sends an empty list. Compare both as empty and keep the stream when the hook left the tool calls alone. --- .../proxy/policy_engine/pipeline_executor.py | 15 +++- .../policy_engine/test_pipeline_executor.py | 84 ++++++++++++++++++- 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 9b0d1e60839..52985d49c7e 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -70,6 +70,10 @@ def _text_snapshot(texts: Sequence[str] | None) -> tuple[str, ...] | None: return None if texts is None else tuple(texts) +def _scanned_texts(texts: Sequence[str] | None) -> tuple[str, ...]: + return tuple(texts or ()) + + def _tool_call_shapes(tool_calls: Sequence[object] | None) -> tuple[tuple[object, object], ...] | None: return None if tool_calls is None else tuple(_tool_call_shape(tool_call) for tool_call in tool_calls) @@ -152,7 +156,9 @@ class _LegacyHookStreamAdapter(CustomGuardrail): and returned ``None`` for, is re-scanned by the same translation so its texts reach the client through the translation's ended-stream write-back. A replacement whose scanned texts do not line up with the originals, or whose tool calls - differ from them, is undeliverable, so the executor releases the original chunks.""" + differ from them, is undeliverable, so the executor releases the original chunks. A stream + that carried no text to scan, such as a tool-only Anthropic message, stays deliverable as + long as the hook left the tool calls alone.""" def __init__( self, @@ -185,15 +191,16 @@ class _LegacyHookStreamAdapter(CustomGuardrail): rewrite: Final = hooked if replacement is None else replacement if rewrite is None: return inputs - scanned: Final = _text_snapshot(inputs.get("texts")) rescanned: Final = await self._rescan(rewrite, logging_obj) - if scanned is None or rescanned is None: + if rescanned is None: raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") rewritten: Final = rescanned.get("texts") - if rewritten is None or len(rewritten) != len(scanned): + if len(_scanned_texts(rewritten)) != len(_scanned_texts(inputs.get("texts"))): raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") if _tool_call_shapes(rescanned.get("tool_calls")) != _tool_call_shapes(inputs.get("tool_calls")): raise UndeliverableStreamRewrite(self.guardrail_name or "unknown") + if not rewritten: + return inputs rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten} return rewritten_inputs diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 2af318c88cd..98674843c73 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1367,6 +1367,44 @@ class _LegacyScanningTranslation: return response +class _ToolOnlyLegacyScanningTranslation(_LegacyScanningTranslation): + """Like the Messages handler on a tool-only message: the ended-stream scan omits "texts" from + the inputs, while the non-streaming scan of the same response sends an empty list.""" + + async def process_output_streaming_response( + self, + responses_so_far, + guardrail_to_apply, + litellm_logging_obj=None, + user_api_key_dict=None, + request_data=None, + deliver_ended_stream_rewrites=False, + ): + request_data.setdefault("response", {"text": "", "tool_calls": [dict(responses_so_far[0]["tool_call"])]}) + await guardrail_to_apply.apply_guardrail( + inputs={"tool_calls": [dict(responses_so_far[0]["tool_call"])]}, + request_data=request_data, + input_type="response", + logging_obj=litellm_logging_obj, + ) + return responses_so_far + + async def process_output_response( + self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None + ): + await guardrail_to_apply.apply_guardrail( + inputs={"texts": [], "tool_calls": list(response.get("tool_calls") or [])}, + request_data={"response": response}, + input_type="response", + logging_obj=litellm_logging_obj, + ) + return response + + +def _tool_only_chunk(): + return {"text": "", "tool_call": _chunk()["tool_call"]} + + def _native(text): return {"native": True, "text": text, "tool_calls": [_chunk()["tool_call"]]} @@ -1375,11 +1413,17 @@ def _legacy_replacement(*texts, tool_calls=None): return {"texts": list(texts), "tool_calls": [_chunk()["tool_call"]] if tool_calls is None else tool_calls} -async def _run_legacy_streaming_step(monkeypatch, guardrail, chunks, on_fail="block", on_error="next"): - return await _run_legacy_streaming_steps(monkeypatch, [guardrail], chunks, on_fail=on_fail, on_error=on_error) +async def _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, on_fail="block", on_error="next", translation=None +): + return await _run_legacy_streaming_steps( + monkeypatch, [guardrail], chunks, on_fail=on_fail, on_error=on_error, translation=translation + ) -async def _run_legacy_streaming_steps(monkeypatch, guardrails, chunks, on_fail="block", on_error="next"): +async def _run_legacy_streaming_steps( + monkeypatch, guardrails, chunks, on_fail="block", on_error="next", translation=None +): monkeypatch.setattr(litellm, "callbacks", list(guardrails)) return await PipelineExecutor.execute_steps( steps=[ @@ -1397,7 +1441,7 @@ async def _run_legacy_streaming_steps(monkeypatch, guardrails, chunks, on_fail=" call_type="completion", policy_name="p", streaming_chunks=chunks, - endpoint_translation=_LegacyScanningTranslation(), + endpoint_translation=_LegacyScanningTranslation() if translation is None else translation, ) @@ -1513,6 +1557,38 @@ async def test_streaming_step_discards_legacy_rewrite_that_drops_the_tool_calls( assert chunks == [_chunk()] +@pytest.mark.asyncio +async def test_streaming_step_passes_a_tool_only_stream_the_legacy_hook_left_alone(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement=None) + chunks = [_tool_only_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, translation=_ToolOnlyLegacyScanningTranslation() + ) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] + assert chunks == [_tool_only_chunk()] + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_discards_a_legacy_tool_call_rewrite_on_a_tool_only_stream(monkeypatch, caplog): + masked_tool_call = {"function": {"name": "lookup", "arguments": '{"ssn": "[MASKED]"}'}} + guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement(tool_calls=[masked_tool_call])) + chunks = [_tool_only_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, translation=_ToolOnlyLegacyScanningTranslation() + ) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_tool_only_chunk()] + + @pytest.mark.asyncio async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(monkeypatch): masker = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world")) From 5ea447e295109b4079d1b41a70f82f1d88024e6a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:19:23 -0700 Subject: [PATCH 146/411] fix(azure): keep the model group segment exact and casefold only deployment names in the relay guard --- litellm/llms/azure/passthrough/transformation.py | 5 ++--- .../passthrough/test_azure_passthrough_transformation.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 69767bb29a3..1b5a4083ebe 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -82,11 +82,10 @@ def foreign_azure_deployment( if match is None: return None deployment: Final = match.group(1) - folded: Final = deployment.casefold() - if folded == model_group.casefold(): + if deployment == model_group: return None served: Final = frozenset(name.casefold() for name in served_models()) - return None if folded in served else deployment + return None if deployment.casefold() in served else deployment def without_api_version(api_base: str) -> str: diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 798ef1f5a4f..c7e86616ee2 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -457,7 +457,7 @@ def test_azure_passthrough_is_streaming_request_reads_the_stream_flag(request_da ("openai/deployments/gpt/chat/completions", None), ("gpt/openai/deployments/gpt-5.4-mini/chat/completions", None), ("gpt/openai/deployments/GPT-5.4-MINI/chat/completions", None), - ("gpt/openai/deployments/Gpt/chat/completions", None), + ("gpt/openai/deployments/Gpt/chat/completions", "Gpt"), ("gpt/models/chat/completions", None), ("gpt/openai/deployments/gpt-5.4/chat/completions", "gpt-5.4"), ("gpt/openai/deployments/other-group/chat/completions", "other-group"), @@ -473,7 +473,7 @@ def test_foreign_azure_deployment_skips_the_router_when_the_segment_is_the_group def served_models(): raise AssertionError("the router must not be consulted for the group's own name") - assert foreign_azure_deployment("gpt/openai/deployments/Gpt/chat/completions", "gpt", served_models) is None + assert foreign_azure_deployment("gpt/openai/deployments/gpt/chat/completions", "gpt", served_models) is None @pytest.mark.parametrize( From 268b944081e7139e54dc88dda6e311152274dc68 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:24:22 -0700 Subject: [PATCH 147/411] fix(spend-tracking): bound the spend-log scan with a statement timeout and name only unanimous alias, team, and owner --- litellm/constants.py | 1 + .../spend_tracking/key_metadata_recovery.py | 62 ++++-- .../test_key_metadata_recovery.py | 176 +++++++++++++----- 3 files changed, 176 insertions(+), 63 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index cfc5b6b86a7..e599a5dd3f9 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1766,6 +1766,7 @@ DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = int(os.getenv("DEFAULT_ACCESS_GROUP_CACH SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600 SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30 SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000 +SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS: Final = 5000 # Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated # callers from forcing a DB query per request for unknown names, while bounding # staleness so a transient DB error (which surfaces as an empty list) cannot diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index d555da99397..29688b61b3d 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -1,7 +1,7 @@ import asyncio from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet -from datetime import datetime +from datetime import datetime, timedelta from types import MappingProxyType from typing import Final, TypeVar @@ -14,6 +14,7 @@ from litellm.constants import ( SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS, SPEND_LOG_KEY_METADATA_CACHE_TTL, SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL, + SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS, ) from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash from litellm.proxy.utils import PrismaClient @@ -36,16 +37,15 @@ ORDER BY token, deleted_at DESC """ _SPEND_LOG_ALIAS_SQL: Final = """ -SELECT DISTINCT ON (api_key) - api_key AS digest, - key_alias, - team_id, - user_id, - MIN(user_id) OVER (PARTITION BY api_key) AS first_owner, - MAX(user_id) OVER (PARTITION BY api_key) AS last_owner +SELECT api_key AS digest, + MIN(key_alias) AS first_alias, + MAX(key_alias) AS last_alias, + MIN(team_id) AS first_team, + MAX(team_id) AS last_team, + MIN(user_id) AS first_owner, + MAX(user_id) AS last_owner FROM ( SELECT api_key, - "startTime", NULLIF(metadata->>'user_api_key_alias', '') AS key_alias, COALESCE(NULLIF(team_id, ''), NULLIF(metadata->>'user_api_key_team_id', '')) AS team_id, COALESCE(NULLIF("user", ''), NULLIF(metadata->>'user_api_key_user_id', '')) AS user_id @@ -55,9 +55,12 @@ FROM ( AND "startTime" < $3::timestamp ) named WHERE COALESCE(key_alias, user_id, team_id) IS NOT NULL -ORDER BY api_key, "startTime" DESC +GROUP BY api_key """ +_SPEND_LOG_STATEMENT_TIMEOUT_SQL: Final = f"SET LOCAL statement_timeout = {SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS}" +_SPEND_LOG_TRANSACTION_TIMEOUT: Final = timedelta(milliseconds=2 * SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS) + _HASHED_JWT_PREFIX: Final = "hashed-jwt-" @@ -75,12 +78,25 @@ class _TokenDigestRow(BaseModel): user_id: str | None = None -class _SpendLogDigestRow(_TokenDigestRow): +def _unanimous(first: str | None, last: str | None) -> str | None: + return first if first == last else None + + +class _SpendLogDigestRow(BaseModel): + digest: str + first_alias: str | None = None + last_alias: str | None = None + first_team: str | None = None + last_team: str | None = None first_owner: str | None = None last_owner: str | None = None - def unanimous_owner(self) -> str | None: - return self.user_id if self.first_owner == self.last_owner else None + def metadata(self) -> KeyMetadataDict: + return KeyMetadataDict( + key_alias=_unanimous(self.first_alias, self.last_alias), + team_id=_unanimous(self.first_team, self.last_team), + user_id=_unanimous(self.first_owner, self.last_owner), + ) _TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...]) @@ -232,14 +248,24 @@ def _cached_spend_log_metadata( ) +async def _spend_log_rows_within_the_statement_timeout( + prisma_client: PrismaClient, + digests: AbstractSet[str], + window: tuple[datetime, datetime], +) -> Sequence[Mapping[str, object]]: + start, end = window + async with prisma_client.db.tx(timeout=_SPEND_LOG_TRANSACTION_TIMEOUT) as transaction: + await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL) + return await transaction.query_raw(_SPEND_LOG_ALIAS_SQL, sorted(digests), start, end) + + async def _query_spend_log_metadata( prisma_client: PrismaClient, digests: AbstractSet[str], window: tuple[datetime, datetime], ) -> Mapping[str, KeyMetadataDict] | None: - start, end = window rows: Final = await _db_or_empty( - lambda: prisma_client.db.query_raw(_SPEND_LOG_ALIAS_SQL, sorted(digests), start, end), + lambda: _spend_log_rows_within_the_statement_timeout(prisma_client, digests, window), "Failed spend-log alias recovery for %d missing keys: %s", len(digests), ) @@ -247,10 +273,10 @@ async def _query_spend_log_metadata( return None return MappingProxyType( { - row.digest: KeyMetadataDict(key_alias=row.key_alias, team_id=row.team_id, user_id=owner) + row.digest: meta for row in _SPEND_LOG_DIGEST_ROWS.validate_python(rows) - for owner in (row.unanimous_owner(),) - if row.digest in digests and (row.key_alias or owner or row.team_id) + for meta in (row.metadata(),) + if row.digest in digests and any(meta.values()) } ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index 682b11f5bb7..89be341c87b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -1,7 +1,7 @@ import asyncio import time from collections.abc import Sequence -from datetime import datetime +from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -9,7 +9,11 @@ import pytest from prisma.errors import PrismaError from litellm.caching.in_memory_cache import InMemoryCache -from litellm.constants import SPEND_LOG_KEY_METADATA_CACHE_TTL, SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL +from litellm.constants import ( + SPEND_LOG_KEY_METADATA_CACHE_TTL, + SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL, + SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS, +) from litellm.proxy.spend_tracking.key_metadata_recovery import ( fill_missing_api_key_aliases, recover_double_hashed_key_metadata, @@ -31,6 +35,26 @@ def _query_raw_spend_logs(rows: Sequence[dict[str, str | None]]) -> AsyncMock: return AsyncMock(side_effect=query_raw) +def _spend_log_row(digest: str, key_alias: str | None, team_id: str | None, user_id: str | None) -> dict[str, str | None]: + return { + "digest": digest, + "first_alias": key_alias, + "last_alias": key_alias, + "first_team": team_id, + "last_team": team_id, + "first_owner": user_id, + "last_owner": user_id, + } + + +def _spend_log_transaction(mock_prisma: MagicMock, query_raw: AsyncMock) -> AsyncMock: + transaction = MagicMock() + transaction.execute_raw = AsyncMock(return_value=0) + transaction.query_raw = query_raw + mock_prisma.db.tx.return_value.__aenter__.return_value = transaction + return query_raw + + def _query_raw_by_table( active_rows: Sequence[dict[str, str | None]], deleted_rows: Sequence[dict[str, str | None]], @@ -240,15 +264,18 @@ async def test_recover_key_metadata_from_spend_logs_resolves_session_token_from_ session_digest = hash_token("cli-session-repro-user-6852") window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs( - [_digest_row(session_digest, "cli-session-repro-user-6852", None, "repro-user-6852")] + query_raw = _spend_log_transaction( + mock_prisma, + _query_raw_spend_logs( + [_spend_log_row(session_digest, "cli-session-repro-user-6852", None, "repro-user-6852")] + ), ) result = await recover_key_metadata_from_spend_logs(mock_prisma, {session_digest}, window, cache=InMemoryCache()) assert result[session_digest]["key_alias"] == "cli-session-repro-user-6852" assert result[session_digest]["user_id"] == "repro-user-6852" - ((_, digests, start, end),) = [call.args for call in mock_prisma.db.query_raw.call_args_list] + ((_, digests, start, end),) = [call.args for call in query_raw.call_args_list] assert digests == [session_digest] assert (start, end) == window @@ -257,19 +284,19 @@ async def test_recover_key_metadata_from_spend_logs_resolves_session_token_from_ async def test_recover_key_metadata_from_spend_logs_skips_query_when_no_missing_keys(): window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) mock_prisma = MagicMock() - mock_prisma.db.query_raw = AsyncMock(return_value=[]) + query_raw = _spend_log_transaction(mock_prisma, AsyncMock(return_value=[])) result = await recover_key_metadata_from_spend_logs(mock_prisma, set(), window, cache=InMemoryCache()) assert result == {} - mock_prisma.db.query_raw.assert_not_called() + query_raw.assert_not_called() @pytest.mark.asyncio async def test_recover_key_metadata_from_spend_logs_returns_empty_on_prisma_error(): window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) mock_prisma = MagicMock() - mock_prisma.db.query_raw = AsyncMock(side_effect=PrismaError("db down")) + query_raw = _spend_log_transaction(mock_prisma, AsyncMock(side_effect=PrismaError("db down"))) result = await recover_key_metadata_from_spend_logs( mock_prisma, {hash_token("cli-session-x")}, window, cache=InMemoryCache() @@ -285,12 +312,15 @@ async def test_recover_key_metadata_from_spend_logs_ignores_foreign_and_all_null foreign = hash_token("cli-session-foreign") window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs( - [ - _digest_row(wanted, "kept-alias", None, "owner-1"), - _digest_row(all_null, None, None, None), - _digest_row(foreign, "foreign-alias", None, "owner-2"), - ] + query_raw = _spend_log_transaction( + mock_prisma, + _query_raw_spend_logs( + [ + _spend_log_row(wanted, "kept-alias", None, "owner-1"), + _spend_log_row(all_null, None, None, None), + _spend_log_row(foreign, "foreign-alias", None, "owner-2"), + ] + ), ) result = await recover_key_metadata_from_spend_logs(mock_prisma, {wanted, all_null}, window, cache=InMemoryCache()) @@ -304,14 +334,14 @@ async def test_recover_key_metadata_from_spend_logs_ignores_foreign_and_all_null async def test_recover_key_metadata_from_spend_logs_skips_non_sha256_keys(): window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) mock_prisma = MagicMock() - mock_prisma.db.query_raw = AsyncMock(return_value=[]) + query_raw = _spend_log_transaction(mock_prisma, AsyncMock(return_value=[])) result = await recover_key_metadata_from_spend_logs( mock_prisma, {"cli-session-raw-1798", "key-hash-short"}, window, cache=InMemoryCache() ) assert result == {} - mock_prisma.db.query_raw.assert_not_called() + query_raw.assert_not_called() @pytest.mark.asyncio @@ -319,13 +349,13 @@ async def test_recover_key_metadata_from_spend_logs_accepts_hashed_jwt_digests() jwt_digest = f"hashed-jwt-{hash_token('jwt-subject-1')}" window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(jwt_digest, None, "team-jwt", "jwt-user")]) + query_raw = _spend_log_transaction(mock_prisma, _query_raw_spend_logs([_spend_log_row(jwt_digest, None, "team-jwt", "jwt-user")])) result = await recover_key_metadata_from_spend_logs(mock_prisma, {jwt_digest}, window, cache=InMemoryCache()) assert result[jwt_digest]["team_id"] == "team-jwt" assert result[jwt_digest]["user_id"] == "jwt-user" - ((_, digests, _, _),) = [call.args for call in mock_prisma.db.query_raw.call_args_list] + ((_, digests, _, _),) = [call.args for call in query_raw.call_args_list] assert digests == [jwt_digest] @@ -336,7 +366,7 @@ async def test_recover_key_metadata_from_spend_logs_serves_repeat_lookups_from_t window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) cache = InMemoryCache() mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(found, "found-alias", None, "owner-1")]) + query_raw = _spend_log_transaction(mock_prisma, _query_raw_spend_logs([_spend_log_row(found, "found-alias", None, "owner-1")])) first = await recover_key_metadata_from_spend_logs(mock_prisma, {found, unknown}, window, cache=cache) second = await recover_key_metadata_from_spend_logs(mock_prisma, {found, unknown}, window, cache=cache) @@ -344,7 +374,7 @@ async def test_recover_key_metadata_from_spend_logs_serves_repeat_lookups_from_t assert first == second assert set(first) == {found} assert first[found]["key_alias"] == "found-alias" - assert mock_prisma.db.query_raw.await_count == 1 + assert query_raw.await_count == 1 @pytest.mark.asyncio @@ -354,15 +384,15 @@ async def test_recover_key_metadata_from_spend_logs_only_queries_digests_the_cac window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) cache = InMemoryCache() mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(cached_digest, "cached-alias", None, None)]) + query_raw = _spend_log_transaction(mock_prisma, _query_raw_spend_logs([_spend_log_row(cached_digest, "cached-alias", None, None)])) await recover_key_metadata_from_spend_logs(mock_prisma, {cached_digest}, window, cache=cache) - mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(new_digest, "new-alias", None, None)]) + query_raw = _spend_log_transaction(mock_prisma, _query_raw_spend_logs([_spend_log_row(new_digest, "new-alias", None, None)])) result = await recover_key_metadata_from_spend_logs(mock_prisma, {cached_digest, new_digest}, window, cache=cache) assert result[cached_digest]["key_alias"] == "cached-alias" assert result[new_digest]["key_alias"] == "new-alias" - ((_, digests, _, _),) = [call.args for call in mock_prisma.db.query_raw.call_args_list] + ((_, digests, _, _),) = [call.args for call in query_raw.call_args_list] assert digests == [new_digest] @@ -371,18 +401,18 @@ async def test_recover_key_metadata_from_spend_logs_rescans_when_the_window_chan digest = hash_token("cli-session-windowed") cache = InMemoryCache() mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs([]) + query_raw = _spend_log_transaction(mock_prisma, _query_raw_spend_logs([])) await recover_key_metadata_from_spend_logs( mock_prisma, {digest}, (datetime(2026, 9, 1), datetime(2026, 9, 4)), cache=cache ) - mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(digest, "later-alias", None, None)]) + query_raw = _spend_log_transaction(mock_prisma, _query_raw_spend_logs([_spend_log_row(digest, "later-alias", None, None)])) result = await recover_key_metadata_from_spend_logs( mock_prisma, {digest}, (datetime(2026, 9, 7), datetime(2026, 9, 10)), cache=cache ) assert result[digest]["key_alias"] == "later-alias" - assert mock_prisma.db.query_raw.await_count == 1 + assert query_raw.await_count == 1 @pytest.mark.asyncio @@ -391,13 +421,13 @@ async def test_recover_key_metadata_from_spend_logs_retries_a_failed_query_only_ window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) cache = InMemoryCache(default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL) mock_prisma = MagicMock() - mock_prisma.db.query_raw = AsyncMock(side_effect=PrismaError("statement timeout")) + query_raw = _spend_log_transaction(mock_prisma, AsyncMock(side_effect=PrismaError("statement timeout"))) started = time.time() assert await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) == {} - mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(digest, "back-online", None, None)]) + query_raw = _spend_log_transaction(mock_prisma, _query_raw_spend_logs([_spend_log_row(digest, "back-online", None, None)])) assert await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=cache) == {} - mock_prisma.db.query_raw.assert_not_awaited() + query_raw.assert_not_awaited() miss_key = next(key for key in cache.ttl_dict if digest in key and not key.endswith(":missed-before")) assert cache.ttl_dict[miss_key] - started <= SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL + 1 cache.ttl_dict[miss_key] = time.time() - 1 @@ -412,14 +442,11 @@ async def test_recover_key_metadata_from_spend_logs_drops_the_owner_of_a_digest_ shared_ui_digest = hash_token("ui-token") window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs( - [ - { - **_digest_row(shared_ui_digest, "ui-token", "litellm-dashboard", "bob"), - "first_owner": "alice", - "last_owner": "bob", - } - ] + query_raw = _spend_log_transaction( + mock_prisma, + _query_raw_spend_logs( + [{**_spend_log_row(shared_ui_digest, "ui-token", "litellm-dashboard", None), "first_owner": "alice", "last_owner": "bob"}] + ), ) result = await recover_key_metadata_from_spend_logs( @@ -434,8 +461,11 @@ async def test_recover_key_metadata_from_spend_logs_keeps_the_owner_when_every_n digest = hash_token("cli-session-one-owner") window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs( - [{**_digest_row(digest, None, None, "carol"), "first_owner": "carol", "last_owner": "carol"}] + query_raw = _spend_log_transaction( + mock_prisma, + _query_raw_spend_logs( + [_spend_log_row(digest, None, None, "carol")] + ), ) result = await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=InMemoryCache()) @@ -450,7 +480,7 @@ async def test_recover_key_metadata_from_spend_logs_forgets_a_miss_long_before_a window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) cache = InMemoryCache(default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL) mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs([_digest_row(found, "found-alias", None, None)]) + query_raw = _spend_log_transaction(mock_prisma, _query_raw_spend_logs([_spend_log_row(found, "found-alias", None, None)])) started = time.time() await recover_key_metadata_from_spend_logs(mock_prisma, {found, unknown}, window, cache=cache) @@ -471,9 +501,9 @@ async def test_recover_key_metadata_from_spend_logs_runs_one_query_for_concurren async def slow_query_raw(sql: str, *params: object) -> list[dict[str, str | None]]: await asyncio.sleep(0.01) - return [_digest_row(digest, "shared-alias", None, None)] + return [_spend_log_row(digest, "shared-alias", None, None)] - mock_prisma.db.query_raw = AsyncMock(side_effect=slow_query_raw) + query_raw = _spend_log_transaction(mock_prisma, AsyncMock(side_effect=slow_query_raw)) results = await asyncio.gather( *( @@ -483,7 +513,7 @@ async def test_recover_key_metadata_from_spend_logs_runs_one_query_for_concurren ) assert all(result[digest]["key_alias"] == "shared-alias" for result in results) - assert mock_prisma.db.query_raw.await_count == 1 + assert query_raw.await_count == 1 @pytest.mark.asyncio @@ -492,7 +522,7 @@ async def test_recover_key_metadata_from_spend_logs_keeps_a_repeated_miss_as_lon window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) cache = InMemoryCache(default_ttl=SPEND_LOG_KEY_METADATA_CACHE_TTL) mock_prisma = MagicMock() - mock_prisma.db.query_raw = _query_raw_spend_logs([]) + query_raw = _spend_log_transaction(mock_prisma, _query_raw_spend_logs([])) await recover_key_metadata_from_spend_logs(mock_prisma, {unknown}, window, cache=cache) first_miss_key = next(key for key in cache.ttl_dict if unknown in key and not key.endswith(":missed-before")) cache.ttl_dict[first_miss_key] = time.time() - 1 @@ -500,5 +530,61 @@ async def test_recover_key_metadata_from_spend_logs_keeps_a_repeated_miss_as_lon await recover_key_metadata_from_spend_logs(mock_prisma, {unknown}, window, cache=cache) - assert mock_prisma.db.query_raw.await_count == 2 + assert query_raw.await_count == 2 assert cache.ttl_dict[first_miss_key] - started >= SPEND_LOG_KEY_METADATA_CACHE_TTL - 1 + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_keeps_the_owner_older_rows_agree_on_when_the_newest_is_nameless(): + digest = hash_token("cli-session-owner-from-older-rows") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + _spend_log_transaction(mock_prisma, _query_raw_spend_logs([_spend_log_row(digest, None, "team-x", "alice")])) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=InMemoryCache()) + + assert result[digest] == {"key_alias": None, "team_id": "team-x", "user_id": "alice"} + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_names_nothing_for_a_field_whose_rows_disagree(): + digest = hash_token("cli-session-disagreeing-rows") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + _spend_log_transaction( + mock_prisma, + _query_raw_spend_logs( + [ + { + **_spend_log_row(digest, None, None, "carol"), + "first_alias": "old-alias", + "last_alias": "renamed-alias", + "first_team": "team-a", + "last_team": "team-b", + } + ] + ), + ) + + result = await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=InMemoryCache()) + + assert result[digest] == {"key_alias": None, "team_id": None, "user_id": "carol"} + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_bounds_the_scan_with_a_statement_timeout(): + digest = hash_token("cli-session-bounded-scan") + window = (datetime(2026, 9, 7), datetime(2026, 9, 10)) + mock_prisma = MagicMock() + calls: list[str] = [] + transaction = MagicMock() + transaction.execute_raw = AsyncMock(side_effect=lambda sql: calls.append(sql) or 0) + transaction.query_raw = AsyncMock(side_effect=lambda sql, *args: calls.append("scan") or []) + mock_prisma.db.tx.return_value.__aenter__.return_value = transaction + + await recover_key_metadata_from_spend_logs(mock_prisma, {digest}, window, cache=InMemoryCache()) + + assert calls == [f"SET LOCAL statement_timeout = {SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS}", "scan"] + assert mock_prisma.db.tx.call_args.kwargs["timeout"] == timedelta( + milliseconds=2 * SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS + ) From 359c26aa1d1403b0dc10c2d7a47b984d13c468c7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:27:46 -0700 Subject: [PATCH 148/411] test(policy_engine): cover the legacy stream paths with no response and no rescan A translation that hands the hook no assembled response leaves the stream as it is, and a rewrite the translation cannot rescan is released as the original stream with a warning. --- .../policy_engine/test_pipeline_executor.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 98674843c73..6de7f7b593d 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1589,6 +1589,68 @@ async def test_streaming_step_discards_a_legacy_tool_call_rewrite_on_a_tool_only assert chunks == [_tool_only_chunk()] +class _ResponselessLegacyScanningTranslation(_LegacyScanningTranslation): + """Like a handler that never stores the assembled response under request_data["response"].""" + + def post_call_hook_response(self, response): + return response + + async def process_output_streaming_response( + self, + responses_so_far, + guardrail_to_apply, + litellm_logging_obj=None, + user_api_key_dict=None, + request_data=None, + deliver_ended_stream_rewrites=False, + ): + await guardrail_to_apply.apply_guardrail( + inputs={"texts": [responses_so_far[0]["text"]]}, + request_data=request_data, + input_type="response", + logging_obj=litellm_logging_obj, + ) + return responses_so_far + + +class _UnscannableRewriteTranslation(_LegacyScanningTranslation): + """Like the chat handler on a response whose choices are plain dicts: the non-streaming scan + never hands anything to the guardrail.""" + + async def process_output_response( + self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None + ): + return response + + +@pytest.mark.asyncio +async def test_streaming_step_leaves_the_stream_alone_when_the_hook_gets_no_response(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail() + chunks = [_chunk()] + + result = await _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, translation=_ResponselessLegacyScanningTranslation() + ) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert guardrail.calls[0]["response"] is None + assert chunks == [_chunk()] + assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_discards_a_legacy_rewrite_the_translation_cannot_rescan(monkeypatch, caplog): + guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement("hello [MASKED]")) + chunks = [_chunk()] + + result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks, translation=_UnscannableRewriteTranslation()) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] + + @pytest.mark.asyncio async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(monkeypatch): masker = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world")) From c6a0381595b3085937e514c7af3f54c2ceb77598 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:38:01 -0700 Subject: [PATCH 149/411] fix(token_counter): bound concurrent HuggingFace encodes so a burst of large counts cannot exhaust memory --- litellm/constants.py | 1 + litellm/litellm_core_utils/token_counter.py | 6 +++- .../litellm_core_utils/test_token_counter.py | 31 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/litellm/constants.py b/litellm/constants.py index a9112c90a54..05d442dd070 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -397,6 +397,7 @@ TOKEN_COUNTER_MAX_EXACT_CHARS: Final = get_env_int_in_range( minimum=1, maximum=1_000_000_000, ) +TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES: Final = 4 MAX_TILE_WIDTH: Final = int(os.getenv("MAX_TILE_WIDTH", 512)) MAX_TILE_HEIGHT: Final = int(os.getenv("MAX_TILE_HEIGHT", 512)) OPENAI_FILE_SEARCH_COST_PER_1K_CALLS: Final = float(os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000)) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index e3af64fd3bf..8624e79faea 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,6 +3,7 @@ import base64 import io import struct +import threading from collections.abc import Callable, Iterable, Mapping, Sequence from typing import Final, Literal, cast @@ -22,6 +23,7 @@ from litellm.constants import ( MAX_TILE_HEIGHT, MAX_TILE_WIDTH, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, + TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES, TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.default_encoding import encoding as default_encoding @@ -320,6 +322,7 @@ Type for a function that counts tokens in a string. """ EXTRAPOLATION_SAMPLES: Final = 16 +_HF_ENCODE_SLOTS: Final = threading.BoundedSemaphore(TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES) def _get_tiktoken_count_function( @@ -587,7 +590,8 @@ def _get_exact_count_function( tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: - return len(tokenizer.encode_batch_fast([text])[0]) + with _HF_ENCODE_SLOTS: + return len(tokenizer.encode_batch_fast([text])[0]) return count_tokens elif tokenizer_json["type"] == "openai_tokenizer": diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index e3dd5feb18d..655be5146cd 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -2,8 +2,10 @@ # This tests litellm.token_counter.token_counter() function import asyncio import importlib +import threading import time import traceback +from concurrent.futures import ThreadPoolExecutor from typing import Final from unittest.mock import MagicMock @@ -16,6 +18,7 @@ import litellm from litellm import create_pretrained_tokenizer, decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants +from litellm.constants import TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( _get_exact_count_function, @@ -162,6 +165,34 @@ def test_count_at_or_below_the_cap_is_exact(): assert count_exactly.call_args_list == [(("a" * 5_000,),)] +class _SlowEncoder: + def __init__(self) -> None: + self._lock: Final = threading.Lock() + self.in_flight = 0 + self.peak_in_flight = 0 + + def encode_batch_fast(self, texts: list[str]) -> list[list[int]]: + with self._lock: + self.in_flight += 1 + self.peak_in_flight = max(self.peak_in_flight, self.in_flight) + time.sleep(0.1) + with self._lock: + self.in_flight -= 1 + return [[0] * len(text) for text in texts] + + +def test_huggingface_counts_run_at_most_the_configured_number_at_once(): + encoder: Final = _SlowEncoder() + count: Final = _get_exact_count_function(None, {"type": "huggingface_tokenizer", "tokenizer": encoder}) + burst: Final = 2 * TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES + + with ThreadPoolExecutor(max_workers=burst) as pool: + counts: Final = tuple(pool.map(count, ["abc"] * burst)) + + assert counts == (3,) * burst + assert 1 < encoder.peak_in_flight <= TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES + + def test_token_counter_applies_the_default_cap(): max_exact_chars: Final = litellm.constants.TOKEN_COUNTER_MAX_EXACT_CHARS prose: Final = ("The quick brown fox jumps over the lazy dog. " * (max_exact_chars // 45 + 1))[:max_exact_chars] From 113350756595b7954d5f830f8a5454895de12526 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:46:55 -0700 Subject: [PATCH 150/411] refactor: type the buffered stream rewrite helpers without Any --- .../llms/anthropic/chat/guardrail_translation/handler.py | 8 ++++---- .../responses/test_openai_responses_guardrail_handler.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index cbbccac17f3..6f966ae1a02 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -13,7 +13,7 @@ Pattern Overview: """ import json -from collections.abc import Mapping, Sequence +from collections.abc import Mapping, MutableSequence, Sequence from copy import deepcopy from dataclasses import dataclass from itertools import chain, repeat @@ -1262,7 +1262,7 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _write_ended_stream_text_rewrite( - responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place rewritten_text: str, ) -> None: """Deliver an ended-stream guardrail text rewrite by rewriting the @@ -1284,7 +1284,7 @@ class AnthropicMessagesHandler(BaseTranslation): @classmethod def _write_ended_stream_tool_call_rewrites( cls, - responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place *, pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], post_guardrail_tool_calls: tuple[_ToolCallShape, ...], @@ -1346,7 +1346,7 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _rewrite_ended_stream_events( - responses_so_far: list[Any], # mutable-ok: rewrites the caller's buffered chunks in place + responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place rewrite_event: _SSEEventRewriter, ) -> None: """Replace every buffered event ``rewrite_event`` returns a rewrite for, in diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index fa1e969b277..6ed4ec6618f 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -17,6 +17,7 @@ from fastapi import HTTPException from openai.types.responses import ResponseFunctionToolCall from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms import get_guardrail_translation_mapping from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, @@ -1238,7 +1239,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: inputs: GenericGuardrailAPIInputs, request_data: dict, input_type: Literal["request", "response"], - logging_obj: Optional[Any] = None, + logging_obj: LiteLLMLoggingObj | None = None, ) -> GenericGuardrailAPIInputs: tool_calls = [ {**tool_call, "function": {**tool_call["function"], "arguments": '{"fruit": "[MASKED]"}'}} From 529b8706ee4a35ca479301242cb07f74e45d6775 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:51:13 -0700 Subject: [PATCH 151/411] test(spend-tracking): mock the spend-log scan through the transaction its caller now opens --- .../test_common_daily_activity.py | 54 +++++++++++-------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index d214ab6b5ac..6cd900cb041 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -518,6 +518,26 @@ async def test_get_api_key_metadata_permanent_miss_never_pages_tokens_or_reads_s assert all("take" not in call.kwargs and "skip" not in call.kwargs for call in token_lookups) +def _spend_log_transaction(mock_prisma: MagicMock, rows: list[dict[str, str | None]]) -> AsyncMock: + transaction = MagicMock() + transaction.execute_raw = AsyncMock(return_value=0) + transaction.query_raw = AsyncMock(return_value=rows) + mock_prisma.db.tx.return_value.__aenter__.return_value = transaction + return transaction.query_raw + + +def _spend_log_row(digest: str, key_alias: str, user_id: str) -> dict[str, str | None]: + return { + "digest": digest, + "first_alias": key_alias, + "last_alias": key_alias, + "first_team": None, + "last_team": None, + "first_owner": user_id, + "last_owner": user_id, + } + + @pytest.mark.asyncio async def test_get_api_key_metadata_permanent_miss_with_a_window_reads_spend_logs_once_within_it(): from litellm.proxy.utils import hash_token @@ -529,14 +549,13 @@ async def test_get_api_key_metadata_permanent_miss_with_a_window_reads_spend_log mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) mock_prisma.db.query_raw = AsyncMock(return_value=[]) + spend_log_query_raw = _spend_log_transaction(mock_prisma, []) result = await get_api_key_metadata(prisma_client=mock_prisma, api_keys={double_hashed}, spend_logs_window=window) assert double_hashed not in result - assert mock_prisma.db.query_raw.await_count == 3 - ((_, digests, start, end),) = [ - call.args for call in mock_prisma.db.query_raw.call_args_list if "LiteLLM_SpendLogs" in call.args[0] - ] + assert mock_prisma.db.query_raw.await_count == 2 + ((_, digests, start, end),) = [call.args for call in spend_log_query_raw.call_args_list] assert digests == [double_hashed] assert (start, end) == window @@ -559,12 +578,10 @@ async def test_get_daily_activity_recovers_a_session_key_alias_from_spend_logs_a return_value=[SimpleNamespace(user_id="session-user", user_email="session@example.com")] ) - async def query_raw(sql, *params): - if "LiteLLM_SpendLogs" in sql: - return [{"digest": session_digest, "key_alias": "cli-session-alias", "team_id": None, "user_id": "session-user"}] - return [] - - mock_prisma.db.query_raw = AsyncMock(side_effect=query_raw) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + spend_log_query_raw = _spend_log_transaction( + mock_prisma, [_spend_log_row(session_digest, "cli-session-alias", "session-user")] + ) result = await get_daily_activity( prisma_client=mock_prisma, @@ -583,9 +600,7 @@ async def test_get_daily_activity_recovers_a_session_key_alias_from_spend_logs_a key_metadata = result.results[0].breakdown.api_keys[session_digest].metadata assert key_metadata.key_alias == "cli-session-alias" assert key_metadata.user_email == "session@example.com" - ((_, digests, start, end),) = [ - call.args for call in mock_prisma.db.query_raw.call_args_list if "LiteLLM_SpendLogs" in call.args[0] - ] + ((_, digests, start, end),) = [call.args for call in spend_log_query_raw.call_args_list] assert digests == [session_digest] assert (start, end) == (datetime(2023, 12, 31), datetime(2024, 1, 3)) @@ -2191,12 +2206,10 @@ async def test_get_api_key_metadata_resolves_session_key_via_spend_log_window(): return_value=[SimpleNamespace(user_id="user-42", user_email="user42@example.com")] ) - async def query_raw(sql, *params): - if "LiteLLM_SpendLogs" in sql: - return [{"digest": session_digest, "key_alias": "cli-session-user-42", "team_id": None, "user_id": "user-42"}] - return [] - - mock_prisma.db.query_raw = AsyncMock(side_effect=query_raw) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) + spend_log_query_raw = _spend_log_transaction( + mock_prisma, [_spend_log_row(session_digest, "cli-session-user-42", "user-42")] + ) result = await get_api_key_metadata( prisma_client=mock_prisma, @@ -2207,8 +2220,7 @@ async def test_get_api_key_metadata_resolves_session_key_via_spend_log_window(): assert result[session_digest]["key_alias"] == "cli-session-user-42" assert result[session_digest]["user_id"] == "user-42" assert result[session_digest]["user_email"] == "user42@example.com" - spend_log_calls = [call.args for call in mock_prisma.db.query_raw.call_args_list if "LiteLLM_SpendLogs" in call.args[0]] - ((_, digests, start, end),) = spend_log_calls + ((_, digests, start, end),) = [call.args for call in spend_log_query_raw.call_args_list] assert digests == [session_digest] assert (start, end) == (datetime(2026, 9, 7), datetime(2026, 9, 10)) From 24ee66328a7bf1a791dc0f7d81051165611877c9 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 17:55:21 -0700 Subject: [PATCH 152/411] test: bound Datadog read-back retries using reset headers --- tests/e2e/logging/datadog_reader.py | 114 ++++++++----- tests/e2e/logging/test_datadog_reader.py | 203 +++++++++++++++++++++++ 2 files changed, 278 insertions(+), 39 deletions(-) diff --git a/tests/e2e/logging/datadog_reader.py b/tests/e2e/logging/datadog_reader.py index 7b4372ef9ba..368c20cb6aa 100644 --- a/tests/e2e/logging/datadog_reader.py +++ b/tests/e2e/logging/datadog_reader.py @@ -12,8 +12,12 @@ empty result. External reads go through ``e2e_http``. from __future__ import annotations +import math +import random import time +from collections.abc import Callable, Mapping from dataclasses import dataclass, field +from typing import Final import pytest from pydantic import BaseModel, ConfigDict, Field @@ -27,12 +31,28 @@ from e2e_config import ( DD_SITE, POLL_TIMEOUT, ) -from e2e_http import URL, Headers, RateLimitedError, Success, post +from e2e_http import URL, Headers, StreamingResponse, send -#: How many rate-limited responses in a row one search tolerates before the -#: hard fail; each retry sleeps a full search interval, so this rides out a -#: burst from a concurrent consumer of the org-wide search budget. -_RATE_LIMIT_RETRIES = 5 +type SearchCall = Callable[[str, float], StreamingResponse] + + +def _seconds(value: str | None) -> float | None: + if value is None: + return None + try: + seconds: Final = float(value) + except ValueError: + return None + return seconds if math.isfinite(seconds) and seconds >= 0 else None + + +def _rate_limit_delay(headers: Mapping[str, str]) -> float: + delays: Final = tuple( + delay + for name in ("x-ratelimit-reset", "retry-after") + if (delay := _seconds(headers.get(name))) is not None + ) + return max(1.0, max(delays, default=DD_SEARCH_INTERVAL)) class _DdAuthHeaders(Headers): @@ -90,6 +110,10 @@ class DdLogsReader: site: str api_key: str = field(repr=False) app_key: str = field(repr=False) + search: SearchCall | None = field(default=None, repr=False) + now: Callable[[], float] = field(default=time.monotonic, repr=False) + sleep: Callable[[float], None] = field(default=time.sleep, repr=False) + jitter: Callable[[], float] = field(default=random.random, repr=False) def events_for_marker(self, marker: str) -> list[DdLogEvent]: """Every ingested event whose attributes carry the marker. DataDog @@ -108,25 +132,28 @@ class DdLogsReader: a single event. A 429 backs off and retries - the search budget is org-wide, so another consumer can empty it under us - while any other failure stays a hard fail.""" - for _ in range(_RATE_LIMIT_RETRIES): - result = post( - URL(f"https://api.{self.site}/api/v2/logs/events/search"), - headers=_DdAuthHeaders(api_key=self.api_key, app_key=self.app_key), - json=_SearchRequest(filter=_SearchFilter(query=query)), - response_type=_SearchResponse, - timeout=30.0, - ) - match result: - case Success(data=page): - return [event.attributes for event in page.data] - case RateLimitedError(retry_after_seconds=retry_after): - time.sleep(retry_after if retry_after else DD_SEARCH_INTERVAL) - case failure: - pytest.fail(f"DataDog Logs Search API at api.{self.site} failed: {failure}") + return self._events_for_query(query, self.now() + POLL_TIMEOUT) + + def _events_for_query(self, query: str, deadline: float) -> list[DdLogEvent]: + search: Final = self.search or self._search_page + while (remaining := deadline - self.now()) > 0: + if (result := search(query, min(30.0, remaining))).ok: + return [event.attributes for event in _SearchResponse.model_validate_json(result.body).data] + if result.status_code != 429: + pytest.fail(f"DataDog Logs Search API at api.{self.site} failed with HTTP {result.status_code}") + if (delay := min(_rate_limit_delay(result.headers) + self.jitter(), deadline - self.now())) > 0: + self.sleep(delay) pytest.fail( - f"DataDog Logs Search API at api.{self.site} still rate-limited after " - f"{_RATE_LIMIT_RETRIES} retries {DD_SEARCH_INTERVAL}s apart - the org-wide " - "logs_public_search_api budget (2 requests per 10s) is exhausted by another consumer" + f"DataDog Logs Search API at api.{self.site} remained rate-limited for {POLL_TIMEOUT}s; " + "the org-wide logs_public_search_api budget is exhausted" + ) + + def _search_page(self, query: str, timeout: float) -> StreamingResponse: + return send( + URL(f"https://api.{self.site}/api/v2/logs/events/search"), + headers=_DdAuthHeaders(api_key=self.api_key, app_key=self.app_key), + json=_SearchRequest(filter=_SearchFilter(query=query)), + timeout=timeout, ) def poll_events_for_marker(self, marker: str) -> list[DdLogEvent]: @@ -140,33 +167,42 @@ class DdLogsReader: hide from the exactly-one assertion - real-DataDog jitter can surface one call's two events tens of seconds apart. Searches pace at DD_SEARCH_INTERVAL, not POLL_INTERVAL, to respect the search API's - request budget. At the deadline the last result is returned as-is.""" - deadline = time.monotonic() + POLL_TIMEOUT - while time.monotonic() < deadline: - events = self.events_for_query(query) + request budget. Discovery, quota retries, and duplicate detection share + one POLL_TIMEOUT deadline; an incomplete settle window fails closed.""" + deadline: Final = self.now() + POLL_TIMEOUT + while (remaining := deadline - self.now()) > 0: + events = self._events_for_query(query, deadline) if events: - return self._settled_events_for_query(query, events) - time.sleep(DD_SEARCH_INTERVAL) - return self.events_for_query(query) + return self._settled_events_for_query(query, events, deadline) + if (remaining := deadline - self.now()) > 0: + self.sleep(min(DD_SEARCH_INTERVAL, remaining)) + return [] - def _settled_events_for_query(self, query: str, events: list[DdLogEvent]) -> list[DdLogEvent]: + def _settled_events_for_query(self, query: str, events: list[DdLogEvent], deadline: float) -> list[DdLogEvent]: """Re-read at every search interval until the settle window closes; a duplicate ends the watch early because more waiting cannot clear it. Keep the last non-empty result: a transient empty search (index lag) must not erase events already confirmed earlier in the settle window. + A successful final search must reach the full settle window before the + shared read-back deadline; otherwise duplicate detection is incomplete. """ - settle_deadline = time.monotonic() + DD_SETTLE_SECONDS + settle_deadline: Final = self.now() + DD_SETTLE_SECONDS last_nonempty = events - while time.monotonic() < settle_deadline: - time.sleep(DD_SEARCH_INTERVAL) - latest = self.events_for_query(query) - if not latest: - continue + if len(events) > 1: + return events + while (remaining := deadline - self.now()) > 0: + self.sleep(min(DD_SEARCH_INTERVAL, remaining)) + if self.now() >= deadline: + break + latest = self._events_for_query(query, deadline) if len(latest) > 1: return latest - last_nonempty = latest - return last_nonempty + if latest: + last_nonempty = latest + if self.now() >= settle_deadline: + return last_nonempty + pytest.fail(f"DataDog log delivery could not complete its duplicate-detection window within {POLL_TIMEOUT}s") def build_dd_logs_reader() -> DdLogsReader: diff --git a/tests/e2e/logging/test_datadog_reader.py b/tests/e2e/logging/test_datadog_reader.py index 00624bd3c84..910a1cefd42 100644 --- a/tests/e2e/logging/test_datadog_reader.py +++ b/tests/e2e/logging/test_datadog_reader.py @@ -1,7 +1,14 @@ +import json +from collections.abc import Iterator, Sequence +from dataclasses import dataclass from typing import Final +import pytest + from datadog_reader import DdLogsReader from datadog_reader import _DdAuthHeaders # pyright: ignore[reportPrivateUsage] # verifies private auth-header serialization +from e2e_config import DD_SEARCH_INTERVAL, POLL_TIMEOUT +from e2e_http import StreamingResponse def test_failure_diagnostics_hide_credentials_without_changing_auth_headers() -> None: @@ -18,3 +25,199 @@ def test_failure_diagnostics_hide_credentials_without_changing_auth_headers() -> "DD-API-KEY": api_key, "DD-APPLICATION-KEY": app_key, } + + +@dataclass +class Clock: + elapsed: float = 0.0 + + def now(self) -> float: + return self.elapsed + + def sleep(self, seconds: float) -> None: + self.elapsed += seconds + + +@dataclass +class Search: + responses: Iterator[StreamingResponse] + calls: tuple[tuple[str, float], ...] = () + + def __call__(self, query: str, timeout: float) -> StreamingResponse: + self.calls += ((query, timeout),) + return next(self.responses) + + +def _page(*event_ids: str) -> StreamingResponse: + return StreamingResponse( + status_code=200, + body=json.dumps({"data": [{"attributes": {"attributes": {"id": event_id}}} for event_id in event_ids]}), + ) + + +def _reader(responses: Sequence[StreamingResponse], clock: Clock) -> tuple[DdLogsReader, Search]: + search: Final = Search(iter(responses)) + return DdLogsReader( + site="us5.datadoghq.com", + api_key="test-api-secret", + app_key="test-app-secret", + search=search, + now=clock.now, + sleep=clock.sleep, + jitter=lambda: 0.25, + ), search + + +def test_429_honors_server_reset_and_preserves_duplicate_events() -> None: + clock: Final = Clock() + reader, search = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": "6"}), _page("first", "duplicate")), + clock, + ) + + events: Final = reader.events_for_query("test-marker") + + assert tuple(event.attributes["id"] for event in events) == ("first", "duplicate") + assert clock.elapsed == 6.25 + assert search.calls == (("test-marker", 30.0), ("test-marker", 30.0)) + + +@pytest.mark.parametrize("reset", ("", "invalid", "nan", "inf", "-1")) +def test_invalid_reset_uses_search_interval(reset: str) -> None: + clock: Final = Clock() + reader, _ = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": reset}), _page()), clock + ) + + assert reader.events_for_query("test-marker") == [] + assert clock.elapsed == DD_SEARCH_INTERVAL + 0.25 + + +def test_zero_reset_cannot_create_a_busy_retry_loop() -> None: + clock: Final = Clock() + reader, _ = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": "0"}), _page()), clock + ) + + assert reader.events_for_query("test-marker") == [] + assert clock.elapsed == 1.25 + + +def test_retry_after_is_not_shortened_by_an_earlier_reset() -> None: + clock: Final = Clock() + reader, _ = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": "2", "retry-after": "8"}), _page()), + clock, + ) + + assert reader.events_for_query("test-marker") == [] + assert clock.elapsed == 8.25 + + +def test_rate_limit_wait_stops_at_deadline_without_issuing_another_request() -> None: + clock: Final = Clock() + reader, search = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT * 10)}),), clock + ) + + with pytest.raises(pytest.fail.Exception, match="remained rate-limited"): + reader.events_for_query("test-marker") + + assert clock.elapsed == POLL_TIMEOUT + assert search.calls == (("test-marker", 30.0),) + + +def test_late_retry_cannot_receive_a_fresh_request_timeout() -> None: + clock: Final = Clock() + reader, search = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT - 5)}), _page()), + clock, + ) + + assert reader.events_for_query("test-marker") == [] + assert search.calls == (("test-marker", 30.0), ("test-marker", 4.75)) + + +@pytest.mark.parametrize("status", (-1, 401, 403, 500)) +def test_non_quota_failures_are_not_retried_or_treated_as_empty_results(status: int) -> None: + clock: Final = Clock() + reader, search = _reader((StreamingResponse(status_code=status, body=""), _page()), clock) + + with pytest.raises(pytest.fail.Exception, match=f"failed with HTTP {status}"): + reader.events_for_query("test-marker") + + assert search.calls == (("test-marker", 30.0),) + assert clock.elapsed == 0 + + +def test_polling_quota_retries_share_the_original_deadline() -> None: + clock: Final = Clock() + reader, search = _reader( + (_page(), StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT)})), + clock, + ) + + with pytest.raises(pytest.fail.Exception, match="remained rate-limited"): + reader.poll_events_for_query("test-marker") + + assert clock.elapsed == POLL_TIMEOUT + assert len(search.calls) == 2 + + +def test_empty_polling_does_not_start_a_final_search_after_its_deadline() -> None: + clock: Final = Clock() + attempts: Final = int(POLL_TIMEOUT / DD_SEARCH_INTERVAL) + reader, search = _reader((_page(),) * attempts, clock) + + assert reader.poll_events_for_query("test-marker") == [] + assert clock.elapsed == POLL_TIMEOUT + assert len(search.calls) == attempts + + +def test_settlement_quota_retries_keep_the_remaining_readback_budget() -> None: + clock: Final = Clock() + empty_reads: Final = int(POLL_TIMEOUT / DD_SEARCH_INTERVAL) - 2 + reader, search = _reader( + (_page(),) * empty_reads + + (_page("first"), StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT)})), + clock, + ) + + with pytest.raises(pytest.fail.Exception, match="remained rate-limited"): + reader.poll_events_for_query("test-marker") + + assert clock.elapsed == POLL_TIMEOUT + assert search.calls[-1] == ("test-marker", DD_SEARCH_INTERVAL) + assert len(search.calls) == empty_reads + 2 + + +def test_settlement_detects_a_duplicate_on_the_final_search() -> None: + clock: Final = Clock() + reader, _ = _reader((_page("first"), _page("first"), _page(), _page("first", "duplicate")), clock) + + events: Final = reader.poll_events_for_query("test-marker") + + assert tuple(event.attributes["id"] for event in events) == ("first", "duplicate") + assert clock.elapsed == 30 + + +def test_settlement_keeps_confirmed_events_through_empty_searches() -> None: + clock: Final = Clock() + reader, _ = _reader((_page("first"), _page(), _page(), _page()), clock) + + events: Final = reader.poll_events_for_query("test-marker") + + assert tuple(event.attributes["id"] for event in events) == ("first",) + assert clock.elapsed == 30 + + +def test_late_delivery_cannot_pass_without_a_complete_settle_window() -> None: + clock: Final = Clock() + empty_reads: Final = int(POLL_TIMEOUT / DD_SEARCH_INTERVAL) - 2 + reader, search = _reader((_page(),) * empty_reads + (_page("first"), _page("first")), clock) + + with pytest.raises(pytest.fail.Exception, match="duplicate-detection window"): + reader.poll_events_for_query("test-marker") + + assert clock.elapsed == POLL_TIMEOUT + assert len(search.calls) == empty_reads + 2 From 84a99dfc3188fdefbf9c39abc87b7621d126328e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:24:51 -0700 Subject: [PATCH 153/411] fix(azure_ai): surface rejected MAI image params as 400 and drop comments --- .../image_generation/mai_transformation.py | 60 +++++++++---------- .../test_mai_image_generation.py | 31 ++++++---- 2 files changed, 48 insertions(+), 43 deletions(-) diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 3421cc0a4a1..2d9d3246b4c 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.exceptions import UnsupportedParamsError from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -21,16 +22,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): DEFAULT_WIDTH = 1024 DEFAULT_HEIGHT = 1024 - # The MAI endpoint produces exactly one image per request. Its documented - # body is model/prompt/width/height (plus `image` for edits) — there is no - # count field, and `n` (or the native `sampleCount`) is accepted and - # ignored, so a request for more silently comes back with one. MAX_IMAGES_PER_REQUEST: Final = 1 - - # Provider-side bounds on the generated image. Both are enforced by the - # MAI endpoint, which 400s with "'width' must be at least 768 pixels." - # Only `size` is checked against them: `width`/`height` pass through - # unmapped, which keeps a future model with different bounds reachable. MIN_DIMENSION_PX: Final = 768 MAX_TOTAL_PX: Final = 1024 * 1024 @@ -158,25 +150,27 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): if k in supported_params: if k == "size" and v: - self._map_size_param(v, optional_params) - elif k == "n" and v is not None and v > self.MAX_IMAGES_PER_REQUEST: + self._map_size_param(v, optional_params, model) + elif k == "n" and v is not None and int(v) > self.MAX_IMAGES_PER_REQUEST: if not drop_params: - raise ValueError( + raise self._unsupported( + model, f"n={v} is not supported for model {model}. The Azure AI MAI image " f"endpoint returns exactly {self.MAX_IMAGES_PER_REQUEST} image per " "request and ignores any count, so a larger value would silently " "return fewer images than requested. Send one request per image, or " - "set drop_params=True to drop n." + "set drop_params=True to drop n.", ) else: optional_params[k] = v elif k in ("width", "height"): optional_params[k] = v elif not drop_params: - raise ValueError( + raise self._unsupported( + model, f"Parameter {k} is not supported for model {model}. " f"Supported parameters are {supported_params} and width/height. " - f"Set drop_params=True to drop unsupported parameters." + f"Set drop_params=True to drop unsupported parameters.", ) if "width" not in optional_params: @@ -187,7 +181,11 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): optional_params.pop("size", None) return optional_params - def _map_size_param(self, size: str, optional_params: dict) -> None: + @staticmethod + def _unsupported(model: str, message: str) -> UnsupportedParamsError: + return UnsupportedParamsError(message=message, llm_provider="azure_ai", model=model) + + def _map_size_param(self, size: str, optional_params: dict, model: str) -> None: size_mapping: Final = { "1024x1024": (1024, 1024), "1792x1024": (1792, 1024), @@ -202,34 +200,32 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): try: width, height = map(int, size.lower().split("x")) except ValueError: - raise ValueError(f"Invalid size format: '{size}'. Expected format 'WIDTHxHEIGHT' (e.g., '1024x1024').") + raise self._unsupported( + model, f"Invalid size format: '{size}'. Expected format 'WIDTHxHEIGHT' (e.g., '1024x1024')." + ) else: - raise ValueError( + raise self._unsupported( + model, f"Unsupported size value: '{size}'. " - f"Use a known size (e.g., '1024x1024') or a custom 'WIDTHxHEIGHT' string." + f"Use a known size (e.g., '1024x1024') or a custom 'WIDTHxHEIGHT' string.", ) - self._validate_dimensions(size=size, width=width, height=height) + self._validate_dimensions(model=model, size=size, width=width, height=height) optional_params["width"] = width optional_params["height"] = height - def _validate_dimensions(self, size: str, width: int, height: int) -> None: - """Reject a `size` the MAI endpoint would 400 on. - - Several OpenAI-standard sizes are outside MAI's bounds: 512x512 and - 256x256 fall under the per-side minimum, and 1792x1024 / 1024x1792 - exceed the total pixel budget. Checking here turns an opaque provider - 400 into an error that names the constraint. - """ + def _validate_dimensions(self, model: str, size: str, width: int, height: int) -> None: if width < self.MIN_DIMENSION_PX or height < self.MIN_DIMENSION_PX: - raise ValueError( + raise self._unsupported( + model, f"Unsupported size value: '{size}'. Azure AI MAI image models require width and " - f"height of at least {self.MIN_DIMENSION_PX} pixels." + f"height of at least {self.MIN_DIMENSION_PX} pixels.", ) if width * height > self.MAX_TOTAL_PX: - raise ValueError( + raise self._unsupported( + model, f"Unsupported size value: '{size}'. Azure AI MAI image models accept at most " - f"{self.MAX_TOTAL_PX} total pixels ({width}x{height} is {width * height})." + f"{self.MAX_TOTAL_PX} total pixels ({width}x{height} is {width * height}).", ) def transform_image_generation_response( diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py index 6cbc79fff2b..29aa5967c9a 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py @@ -4,6 +4,7 @@ import httpx import pytest import litellm +from litellm.exceptions import UnsupportedParamsError from litellm.llms.azure.azure import AzureChatCompletion from litellm.llms.azure.image_generation import get_azure_image_generation_config from litellm.llms.azure.image_generation.http_utils import ( @@ -180,7 +181,7 @@ class TestAzureMAIImageGeneration: def test_map_openai_params_unsupported_size_raises(self): config = AzureFoundryMAIImageGenerationConfig() - with pytest.raises(ValueError, match="Unsupported size value: 'auto'"): + with pytest.raises(UnsupportedParamsError, match="Unsupported size value: 'auto'"): config.map_openai_params( non_default_params={"size": "auto"}, optional_params={}, @@ -190,7 +191,7 @@ class TestAzureMAIImageGeneration: def test_map_openai_params_invalid_custom_size_raises(self): config = AzureFoundryMAIImageGenerationConfig() - with pytest.raises(ValueError, match="Invalid size format: '1024xabc'"): + with pytest.raises(UnsupportedParamsError, match="Invalid size format: '1024xabc'"): config.map_openai_params( non_default_params={"size": "1024xabc"}, optional_params={}, @@ -200,9 +201,8 @@ class TestAzureMAIImageGeneration: @pytest.mark.parametrize("size", ["512x512", "256x256", "700x1400"]) def test_map_openai_params_size_below_minimum_dimension_raises(self, size): - """MAI requires >= 768px per side; the OpenAI size table offered smaller ones.""" config = AzureFoundryMAIImageGenerationConfig() - with pytest.raises(ValueError, match="at least 768 pixels"): + with pytest.raises(UnsupportedParamsError, match="at least 768 pixels"): config.map_openai_params( non_default_params={"size": size}, optional_params={}, @@ -212,9 +212,8 @@ class TestAzureMAIImageGeneration: @pytest.mark.parametrize("size", ["1792x1024", "1024x1792"]) def test_map_openai_params_size_over_total_pixel_budget_raises(self, size): - """MAI caps total pixels at 1024*1024, so both landscape/portrait sizes 400 upstream.""" config = AzureFoundryMAIImageGenerationConfig() - with pytest.raises(ValueError, match="at most 1048576 total pixels"): + with pytest.raises(UnsupportedParamsError, match="at most 1048576 total pixels"): config.map_openai_params( non_default_params={"size": size}, optional_params={}, @@ -223,7 +222,6 @@ class TestAzureMAIImageGeneration: ) def test_map_openai_params_explicit_width_height_not_range_checked(self): - """width/height pass through unmapped, so a future model's bounds stay reachable.""" config = AzureFoundryMAIImageGenerationConfig() optional_params = config.map_openai_params( non_default_params={"width": 1792, "height": 1024}, @@ -234,11 +232,10 @@ class TestAzureMAIImageGeneration: assert optional_params["width"] == 1792 assert optional_params["height"] == 1024 - @pytest.mark.parametrize("n", [2, 4]) + @pytest.mark.parametrize("n", [2, 4, "2"]) def test_map_openai_params_multi_image_n_raises(self, n): - """The MAI endpoint returns one image and ignores any count, so n>1 must not pass silently.""" config = AzureFoundryMAIImageGenerationConfig() - with pytest.raises(ValueError, match="returns exactly 1 image per request"): + with pytest.raises(UnsupportedParamsError, match="returns exactly 1 image per request"): config.map_openai_params( non_default_params={"n": n}, optional_params={}, @@ -266,9 +263,21 @@ class TestAzureMAIImageGeneration: ) assert optional_params["n"] == 1 + @pytest.mark.parametrize("params", [{"n": 2}, {"size": "512x512"}, {"size": "1792x1024"}]) + def test_image_generation_rejected_params_surface_as_400(self, params): + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.image_generation( + model="azure_ai/MAI-Image-2.5", + prompt="A photograph of a red fox", + api_key="test-key", + api_base="https://my-resource.services.ai.azure.com", + **params, + ) + assert exc_info.value.status_code == 400 + def test_map_openai_params_unsupported_param_raises(self): config = AzureFoundryMAIImageGenerationConfig() - with pytest.raises(ValueError, match="Parameter quality is not supported"): + with pytest.raises(UnsupportedParamsError, match="Parameter quality is not supported"): config.map_openai_params( non_default_params={"quality": "hd"}, optional_params={}, From 38a3de764154a600441a37c746d78c7ca89d57bf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:46:18 -0700 Subject: [PATCH 154/411] fix(azure_ai): use the pixel cap the live MAI endpoint enforces --- .../image_generation/mai_transformation.py | 2 +- .../test_mai_image_generation.py | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 2d9d3246b4c..7f7a1044689 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -24,7 +24,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): MAX_IMAGES_PER_REQUEST: Final = 1 MIN_DIMENSION_PX: Final = 768 - MAX_TOTAL_PX: Final = 1024 * 1024 + MAX_TOTAL_PX: Final = 1_056_768 @staticmethod def get_mai_image_generation_url( diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py index 29aa5967c9a..a0bc3fb00b7 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py @@ -213,7 +213,7 @@ class TestAzureMAIImageGeneration: @pytest.mark.parametrize("size", ["1792x1024", "1024x1792"]) def test_map_openai_params_size_over_total_pixel_budget_raises(self, size): config = AzureFoundryMAIImageGenerationConfig() - with pytest.raises(UnsupportedParamsError, match="at most 1048576 total pixels"): + with pytest.raises(UnsupportedParamsError, match="at most 1056768 total pixels"): config.map_openai_params( non_default_params={"size": size}, optional_params={}, @@ -221,6 +221,27 @@ class TestAzureMAIImageGeneration: drop_params=True, ) + @pytest.mark.parametrize("size", ["1032x1024", "1376x768"]) + def test_map_openai_params_size_at_live_pixel_cap_passes_through(self, size): + config = AzureFoundryMAIImageGenerationConfig() + optional_params = config.map_openai_params( + non_default_params={"size": size}, + optional_params={}, + model="MAI-Image-2.5", + drop_params=False, + ) + assert optional_params["width"] * optional_params["height"] == 1_056_768 + + def test_map_openai_params_size_one_pixel_over_live_cap_raises(self): + config = AzureFoundryMAIImageGenerationConfig() + with pytest.raises(UnsupportedParamsError, match="at most 1056768 total pixels"): + config.map_openai_params( + non_default_params={"size": "1033x1024"}, + optional_params={}, + model="MAI-Image-2.5", + drop_params=False, + ) + def test_map_openai_params_explicit_width_height_not_range_checked(self): config = AzureFoundryMAIImageGenerationConfig() optional_params = config.map_openai_params( From 1a9c6ce390aeaf6835351cbd2169eca5c972e4d3 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 8 Sep 2026 19:01:01 -0700 Subject: [PATCH 155/411] fix(mcp): preserve proxy logging and authorization coverage (#40337) * test(mcp): exercise /mcp/proxy authorization against the real registry instead of patched manager methods * fix(mcp): preserve proxy logging and authorization coverage * test(mcp): respect the proxy FastAPI import boundary --- .../proxy/_experimental/mcp_server/server.py | 25 +- tests/mcp_tests/mcp_server.py | 15 +- .../test_configs/test_config_mcp_e2e.yaml | 3 + tests/mcp_tests/test_proxy_mcp_e2e.py | 388 +++++++++++++++--- .../mcp_server/test_mcp_proxy_mode.py | 353 ++-------------- 5 files changed, 384 insertions(+), 400 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 44c00df996c..30793ef246d 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -767,12 +767,12 @@ if MCP_AVAILABLE: _stateful_auth_context_cleanup_task.cancel() with contextlib.suppress(asyncio.CancelledError): await _stateful_auth_context_cleanup_task - if _session_manager_cm: - await _session_manager_cm.__aexit__(None, None, None) - if _session_manager_stateful_cm: - await _session_manager_stateful_cm.__aexit__(None, None, None) if _sse_session_manager_cm: await _sse_session_manager_cm.__aexit__(None, None, None) + if _session_manager_stateful_cm: + await _session_manager_stateful_cm.__aexit__(None, None, None) + if _session_manager_cm: + await _session_manager_cm.__aexit__(None, None, None) except Exception as e: verbose_logger.exception("Error during session manager shutdown: %s", e) @@ -1005,6 +1005,7 @@ if MCP_AVAILABLE: if _mcp_proxy_mode.get() and name in MCP_PROXY_TOOL_NAMES: assert user_api_key_auth is not None + proxy_call_start: Final = datetime.now() # noqa: DTZ005 # logging pipeline uses naive datetimes proxy_logging_obj: Final = ( await _build_virtual_call_logging_obj( name=name, @@ -1016,7 +1017,7 @@ if MCP_AVAILABLE: if name == MCP_PROXY_CALL_TOOL_NAME else None ) - return await handle_mcp_proxy_tool( + proxy_result: Final = await handle_mcp_proxy_tool( name=name, arguments=arguments or {}, # mutable-ok: proxy handler payload user_api_key_dict=user_api_key_auth, @@ -1028,6 +1029,16 @@ if MCP_AVAILABLE: raw_headers=raw_headers, litellm_logging_obj=proxy_logging_obj, ) + if proxy_logging_obj is not None: + return await _fire_mcp_tool_call_logging( + logging_obj=proxy_logging_obj, + result=proxy_result, + start_time=proxy_call_start, + end_time=datetime.now(), # noqa: DTZ005 # matches the logging pipeline start time + user_api_key_auth=user_api_key_auth, + request_data=types.MappingProxyType({"name": name, "arguments": arguments}), + ) + return proxy_result if name not in VIRTUAL_TOOL_NAMES: return None @@ -3493,7 +3504,9 @@ if MCP_AVAILABLE: server_name: str | None, session_id: str | None = None, ) -> StandardLoggingMCPToolCall: - mcp_server: Final = global_mcp_server_manager._get_mcp_server_from_tool_name(name) + mcp_server: Final = global_mcp_server_manager._get_mcp_server_from_tool_name( + add_server_prefix_to_name(name, server_name) if server_name else name + ) namespaced_tool_name: Final = f"{server_name}/{name}" if server_name else name if mcp_server: mcp_info: Final = mcp_server.mcp_info or {} diff --git a/tests/mcp_tests/mcp_server.py b/tests/mcp_tests/mcp_server.py index bc6accbb721..eba7cae1bca 100644 --- a/tests/mcp_tests/mcp_server.py +++ b/tests/mcp_tests/mcp_server.py @@ -1,10 +1,12 @@ # math_server.py import argparse import os +from typing import Final -from mcp.server.fastmcp import FastMCP +from mcp.server.fastmcp import Context, FastMCP mcp = FastMCP("Math") +ADD_OFFSET: Final = int(os.getenv("MCP_ADD_OFFSET", "0")) def _parse_args() -> argparse.Namespace: @@ -31,7 +33,7 @@ def _parse_args() -> argparse.Namespace: @mcp.tool() def add(a: int, b: int) -> int: """Add two numbers""" - return a + b + return a + b + ADD_OFFSET @mcp.tool() @@ -40,6 +42,15 @@ def multiply(a: int, b: int) -> int: return a * b +@mcp.tool() +def request_headers(ctx: Context) -> dict[str, str]: + request: Final = ctx.request_context.request + return { + "authorization": request.headers.get("authorization", "") if request is not None else "", + "x-request-tag": request.headers.get("x-request-tag", "") if request is not None else "", + } + + def main() -> None: args = _parse_args() transport = (args.transport or "stdio").lower() diff --git a/tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml b/tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml index ad68a03781d..19fad3d1393 100644 --- a/tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml +++ b/tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml @@ -23,3 +23,6 @@ mcp_servers: transport: http url: http://127.0.0.1:0/mcp allow_all_keys: true + math_restricted: + transport: http + url: http://127.0.0.1:0/mcp diff --git a/tests/mcp_tests/test_proxy_mcp_e2e.py b/tests/mcp_tests/test_proxy_mcp_e2e.py index a97eed82e18..5e5b9db2dc6 100644 --- a/tests/mcp_tests/test_proxy_mcp_e2e.py +++ b/tests/mcp_tests/test_proxy_mcp_e2e.py @@ -1,27 +1,39 @@ import asyncio import json import os +import queue import socket import subprocess import sys +import tempfile import threading import time import typing +from contextlib import asynccontextmanager, contextmanager +from dataclasses import dataclass +from datetime import datetime from pathlib import Path +import httpx import pytest import uvicorn import yaml from mcp import ClientSession from mcp.client.streamable_http import streamablehttp_client +from mcp.types import CallToolResult +from starlette.requests import Request +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_proxy_tool +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, ProxyException, UserAPIKeyAuth from litellm.proxy.proxy_server import ( app as proxy_app, +) +from litellm.proxy.proxy_server import ( cleanup_router_config_variables, initialize, ) - CONFIG_TEMPLATE_PATH = Path("tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml") MCP_SERVER_SCRIPT = Path("tests/mcp_tests/mcp_server.py") PROJECT_ROOT = Path(__file__).resolve().parents[2] @@ -46,28 +58,49 @@ def _clear_proxy_database_env() -> typing.Iterator[None]: mp.undo() -def _initialize_proxy(config_path: str) -> None: +async def _initialize_proxy(config_path: str) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + cleanup_router_config_variables() - asyncio.run(initialize(config=config_path, debug=True)) + await initialize(config=config_path, debug=True) + for server_id, upstream in tuple(global_mcp_server_manager.registry.items()): + if upstream.server_name != "math_restricted": + continue + global_mcp_server_manager.registry[server_id] = upstream.model_copy( + update={"tool_name_to_display_name": {"add": "Add Numbers"}} + ) + + +@dataclass(frozen=True) +class ProxyRig: + url: str + config_path: str + loop: asyncio.AbstractEventLoop def _start_proxy_server( config_path: str, -) -> tuple[str, uvicorn.Server, threading.Thread, socket.socket]: - _initialize_proxy(config_path) - +) -> tuple[ProxyRig, uvicorn.Server, threading.Thread, socket.socket]: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind(("127.0.0.1", 0)) host, port = sock.getsockname() - config = uvicorn.Config(proxy_app, host=host, port=port, log_level="warning") + config = uvicorn.Config(proxy_app, host=host, port=port, log_level="warning", lifespan="off") server = uvicorn.Server(config) + loop = asyncio.new_event_loop() + + async def _serve() -> None: + from litellm.proxy._experimental.mcp_server import server as mcp_server + + await _initialize_proxy(config_path) + async with proxy_app.router.lifespan_context(proxy_app), mcp_server.lifespan(proxy_app): + await server.serve(sockets=[sock]) + def _run() -> None: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.run_until_complete(server.serve(sockets=[sock])) + with asyncio.Runner(loop_factory=lambda: loop) as runner: + runner.run(_serve()) thread = threading.Thread(target=_run, daemon=True) thread.start() @@ -80,75 +113,93 @@ def _start_proxy_server( raise TimeoutError("Proxy server did not start in time") time.sleep(0.05) - return f"http://{host}:{port}", server, thread, sock + return ProxyRig(f"http://{host}:{port}", config_path, loop), server, thread, sock -@pytest.fixture(scope="session") -def math_streamable_http_server() -> str: +@contextmanager +def _math_http_server(offset: int) -> typing.Iterator[str]: host = "127.0.0.1" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind((host, 0)) _, port = sock.getsockname() - cmd = [ - sys.executable, - str(MCP_SERVER_SCRIPT), - "--transport", - "http", - "--host", - host, - "--port", - str(port), - ] - - env = os.environ.copy() - server_process = subprocess.Popen( - cmd, - cwd=str(PROJECT_ROOT), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - start_time = time.time() - while True: - if server_process.poll() is not None: - stdout, stderr = server_process.communicate() - raise RuntimeError( - f"Streamable HTTP MCP server exited early.\nSTDOUT: {stdout.decode()}\nSTDERR: {stderr.decode()}" - ) + with tempfile.TemporaryFile() as server_log: + process = subprocess.Popen( + [sys.executable, str(MCP_SERVER_SCRIPT), "--transport", "http", "--host", host, "--port", str(port)], + cwd=str(PROJECT_ROOT), + stdout=server_log, + stderr=subprocess.STDOUT, + env={**os.environ, "MCP_ADD_OFFSET": str(offset)}, + ) try: - with socket.create_connection((host, port), timeout=0.1): - break - except OSError: - if time.time() - start_time > PROXY_START_TIMEOUT: - server_process.terminate() - raise TimeoutError("Streamable HTTP MCP server did not start in time") - time.sleep(0.05) - - yield f"http://{host}:{port}" - - server_process.terminate() - try: - server_process.wait(timeout=5) - except subprocess.TimeoutExpired: - server_process.kill() + start_time = time.monotonic() + while True: + if process.poll() is not None: + server_log.seek(0) + raise RuntimeError(f"MCP upstream exited early: {server_log.read().decode()}") + try: + with socket.create_connection((host, port), timeout=0.1): + break + except OSError: + if time.monotonic() - start_time > PROXY_START_TIMEOUT: + raise TimeoutError("Streamable HTTP MCP server did not start in time") + time.sleep(0.05) + yield f"http://{host}:{port}" + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) @pytest.fixture(scope="session") -def proxy_server_url(tmp_path_factory: pytest.TempPathFactory, math_streamable_http_server: str): +def math_streamable_http_server() -> typing.Iterator[str]: + with _math_http_server(100) as url: + yield url + + +@pytest.fixture(scope="session") +def math_restricted_server() -> typing.Iterator[str]: + with _math_http_server(200) as url: + yield url + + +@pytest.fixture(scope="session") +def _proxy_server( + tmp_path_factory: pytest.TempPathFactory, + math_streamable_http_server: str, + math_restricted_server: str, +): config_dir = tmp_path_factory.mktemp("mcp_e2e") config_path = config_dir / "config.yaml" config = yaml.safe_load(CONFIG_TEMPLATE_PATH.read_text()) + config["mcp_servers"]["math_stdio"]["command"] = sys.executable config["mcp_servers"]["math_streamable_http"]["url"] = f"{math_streamable_http_server}/mcp" + config["mcp_servers"]["math_restricted"]["url"] = f"{math_restricted_server}/mcp" + config["general_settings"]["custom_auth"] = f"{__name__}.authorize_proxy_key" + config["litellm_settings"]["callbacks"] = [f"{__name__}.proxy_call_recorder"] + config["mcp_servers"]["math_restricted"]["mcp_info"] = {"mcp_server_cost_info": {"default_cost_per_query": 0.25}} config_path.write_text(yaml.safe_dump(config)) - server_url, server, thread, sock = _start_proxy_server(str(config_path)) + rig, server, thread, sock = _start_proxy_server(str(config_path)) - yield server_url + try: + yield rig + finally: + server.should_exit = True + thread.join(timeout=10) + sock.close() + assert not thread.is_alive(), "Proxy did not shut down" - server.should_exit = True - thread.join(timeout=10) - sock.close() + +@pytest.fixture +def proxy_server_url(_proxy_server: ProxyRig, setup_and_teardown: None) -> str: + asyncio.run_coroutine_threadsafe(_initialize_proxy(_proxy_server.config_path), _proxy_server.loop).result( + timeout=30 + ) + return _proxy_server.url class TestProxyMcpSimpleConnections: @@ -192,7 +243,7 @@ class TestProxyMcpSimpleConnections: assert result.content first_content = result.content[0] text = getattr(first_content, "text", None) - assert text == "11" + assert text == "111" @pytest.mark.asyncio async def test_proxy_mcp_lists_all_servers_without_header(self, proxy_server_url: str) -> None: @@ -222,7 +273,7 @@ class TestProxyMcpSimpleConnections: stdio_result = await _call_and_get_text("math_stdio-add", a=2, b=3) streamable_result = await _call_and_get_text("math_streamable_http-add", a=4, b=5) assert stdio_result == "5" - assert streamable_result == "9" + assert streamable_result == "109" class TestProxyMcpStatelessBehavior: @@ -349,7 +400,7 @@ class TestProxyMcpSchemaDiscoveryMode: }, ) assert stdio.isError is False and stdio.content[0].text == "7" - assert http.isError is False and http.content[0].text == "11" + assert http.isError is False and http.content[0].text == "111" @pytest.mark.asyncio async def test_server_scope_header_narrows_discovery(self, proxy_server_url: str) -> None: @@ -384,6 +435,12 @@ class TestProxyMcpSchemaDiscoveryMode: stale = await session.call_tool("get_tool_schema", arguments={"tool_id": "0" * 32}) assert stale.isError is True and "unauthorized tool_id" in stale.content[0].text + for not_an_object in ("wrong", False): + refused_args = await session.call_tool( + "call_tool", arguments={"tool_id": tool_id, "arguments": not_an_object} + ) + assert refused_args.isError is True and "object" in refused_args.content[0].text + direct = await session.call_tool("math_stdio-add", arguments={"a": 1, "b": 2}) assert direct.isError is True and "unavailable on /mcp/proxy" in direct.content[0].text @@ -391,3 +448,208 @@ class TestProxyMcpSchemaDiscoveryMode: with pytest.raises(McpError) as refused: await operation() assert refused.value.error.code == METHOD_NOT_FOUND + + +async def authorize_proxy_key(request: Request, api_key: str) -> UserAPIKeyAuth: + permissions = { + "sk-1234": LiteLLM_ObjectPermissionTable(object_permission_id="open", mcp_servers=["math_stdio"]), + "sk-restricted": LiteLLM_ObjectPermissionTable( + object_permission_id="restricted", mcp_servers=["math_restricted"] + ), + "sk-none": LiteLLM_ObjectPermissionTable(object_permission_id="none", mcp_servers=["no-mcp-servers"]), + "sk-add-only": LiteLLM_ObjectPermissionTable( + object_permission_id="add-only", mcp_servers=["math_stdio"], mcp_tool_permissions={"math_stdio": ["add"]} + ), + } + permission = permissions.get(api_key) + if permission is None: + raise ProxyException(message="Unknown test key", type="authentication_error", param=None, code=401) + return UserAPIKeyAuth(api_key=api_key, user_id=api_key, object_permission=permission) + + +class ProxyCallRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: queue.Queue[str] = queue.Queue() + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + payload = kwargs.get("standard_logging_object") + if isinstance(payload, dict) and payload.get("call_type") == "call_mcp_tool": + self.events.put(json.dumps(payload, default=str)) + + +proxy_call_recorder = ProxyCallRecorder() + + +@asynccontextmanager +async def _scoped_session(url: str, key: str = "sk-1234", **headers: str) -> typing.AsyncIterator[ClientSession]: + async with asyncio.timeout(30): + async with _proxy_session(url, Authorization=f"Bearer {key}", **headers) as (read, write, _sid): + async with ClientSession(read, write) as session: + await session.initialize() + yield session + + +async def _search(session: ClientSession, query: str) -> dict[str, str]: + result = await session.call_tool("search_tools", arguments={"query": query}) + assert result.isError is False, result + return {hit["name"]: hit["tool_id"] for hit in _payload(result)} + + +async def _call(session: ClientSession, tool_id: str, a: int = 3, b: int = 4) -> CallToolResult: + return await session.call_tool("call_tool", arguments={"tool_id": tool_id, "arguments": {"a": a, "b": b}}) + + +def _assert_unauthorized(result: CallToolResult) -> None: + assert result.isError is True + assert result.content[0].text == "Unknown or unauthorized tool_id" + + +class TestProxyMcpAuthorizationScope: + @pytest.mark.asyncio + async def test_server_grant_bounds_search_and_blocks_foreign_ids(self, proxy_server_url: str) -> None: + async with _scoped_session(proxy_server_url, "sk-restricted") as granted: + restricted_id = (await _search(granted, "add"))["math_restricted-add"] + assert (await _call(granted, restricted_id)).content[0].text == "207" + async with _scoped_session(proxy_server_url) as ungranted: + assert set(await _search(ungranted, "add")) == {"math_stdio-add", "math_streamable_http-add"} + _assert_unauthorized(await ungranted.call_tool("get_tool_schema", {"tool_id": restricted_id})) + _assert_unauthorized(await _call(ungranted, restricted_id)) + + @pytest.mark.asyncio + async def test_no_mcp_servers_sentinel_hides_every_tool(self, proxy_server_url: str) -> None: + async with _scoped_session(proxy_server_url) as granted: + tool_id = (await _search(granted, "add"))["math_stdio-add"] + async with _scoped_session(proxy_server_url, "sk-none") as session: + assert await _search(session, "add") == {} + _assert_unauthorized(await session.call_tool("get_tool_schema", {"tool_id": tool_id})) + _assert_unauthorized(await _call(session, tool_id)) + + @pytest.mark.asyncio + async def test_tool_grant_hides_ungranted_tools_and_blocks_their_ids(self, proxy_server_url: str) -> None: + async with _scoped_session(proxy_server_url) as granted: + multiply_id = (await _search(granted, "multiply"))["math_stdio-multiply"] + async with _scoped_session(proxy_server_url, "sk-add-only", **{"x-mcp-servers": "math_stdio"}) as session: + ids = await _search(session, "add multiply request_headers") + assert set(ids) == {"math_stdio-add"} + assert (await _call(session, ids["math_stdio-add"])).content[0].text == "7" + _assert_unauthorized(await session.call_tool("get_tool_schema", {"tool_id": multiply_id})) + _assert_unauthorized(await _call(session, multiply_id)) + + @pytest.mark.asyncio + async def test_same_named_tools_keep_distinct_ids_and_reach_their_own_upstream(self, proxy_server_url: str) -> None: + async with _scoped_session(proxy_server_url, "sk-restricted") as session: + ids = await _search(session, "add") + assert set(ids) == {"math_stdio-add", "math_streamable_http-add", "math_restricted-add"} + assert len(set(ids.values())) == 3 + assert all(len(tool_id) == 32 for tool_id in ids.values()) + for name, expected in ( + ("math_stdio-add", "7"), + ("math_streamable_http-add", "107"), + ("math_restricted-add", "207"), + ): + schema = _payload(await session.call_tool("get_tool_schema", {"tool_id": ids[name]})) + assert schema["name"] == name + assert schema["tool_id"] == ids[name] + result = await _call(session, ids[name]) + assert result.isError is False + assert result.content[0].text == expected + + @pytest.mark.asyncio + async def test_server_scope_header_narrows_grants_and_blocks_out_of_scope_ids(self, proxy_server_url: str) -> None: + async with _scoped_session(proxy_server_url, "sk-restricted") as unscoped: + other_id = (await _search(unscoped, "add"))["math_stdio-add"] + async with _scoped_session( + proxy_server_url, "sk-restricted", **{"x-mcp-servers": "math_restricted"} + ) as session: + ids = await _search(session, "add") + assert set(ids) == {"math_restricted-add"} + _assert_unauthorized(await session.call_tool("get_tool_schema", {"tool_id": other_id})) + _assert_unauthorized(await _call(session, other_id)) + assert (await _call(session, ids["math_restricted-add"])).content[0].text == "207" + + @pytest.mark.asyncio + @pytest.mark.parametrize("key", [None, "sk-invalid"]) + async def test_missing_or_invalid_key_cannot_initialize(self, proxy_server_url: str, key: str | None) -> None: + async with httpx.AsyncClient() as client: + response = await client.post( + f"{proxy_server_url}/mcp/proxy", + headers={ + "Accept": "application/json, text/event-stream", + **({"Authorization": f"Bearer {key}"} if key else {}), + }, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "auth-test", "version": "1"}, + }, + }, + ) + assert response.status_code == 401, response.text + + @pytest.mark.asyncio + async def test_server_headers_are_forwarded_only_to_the_named_upstream(self, proxy_server_url: str) -> None: + for tag in ("first-request", "second-request"): + async with _scoped_session( + proxy_server_url, + "sk-restricted", + **{ + "x-mcp-math_restricted-authorization": f"Bearer {tag}", + "x-mcp-math_restricted-x-request-tag": tag, + }, + ) as session: + ids = await _search(session, "request_headers") + for name, expected in ( + ("math_restricted", {"authorization": f"Bearer {tag}", "x-request-tag": tag}), + ("math_streamable_http", {"authorization": "", "x-request-tag": ""}), + ): + result = await session.call_tool( + "call_tool", {"tool_id": ids[f"{name}-request_headers"], "arguments": {}} + ) + assert result.isError is False + assert _payload(result) == expected + + @pytest.mark.asyncio + async def test_proxy_call_emits_spend_log(self, proxy_server_url: str) -> None: + async with _scoped_session(proxy_server_url, "sk-restricted") as session: + tool_id = (await _search(session, "add"))["math_restricted-add"] + result = await _call(session, tool_id, 123, 456) + assert result.isError is False and result.content[0].text == "779" + async with asyncio.timeout(10): + while True: + payload = json.loads(await asyncio.to_thread(proxy_call_recorder.events.get, True, 5)) + if payload.get("metadata", {}).get("mcp_tool_call_metadata", {}).get("arguments") == { + "a": 123, + "b": 456, + }: + break + assert payload["call_type"] == "call_mcp_tool" + assert payload["response_cost"] == 0.25 + assert payload["status"] == "success" + assert payload["metadata"]["mcp_tool_call_metadata"]["mcp_server_name"] == "math_restricted" + assert payload["metadata"]["mcp_tool_call_metadata"]["name"] == "add" + assert payload["metadata"]["mcp_tool_call_metadata"]["namespaced_tool_name"] == "math_restricted/add" + + @pytest.mark.parametrize("arguments", ["wrong", False, None, [], 0]) + def test_handler_rejects_non_object_arguments( + self, proxy_server_url: str, _proxy_server: ProxyRig, arguments: object + ) -> None: + async def check() -> None: + auth = UserAPIKeyAuth( + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="validation", mcp_servers=["math_stdio"] + ) + ) + hits = _payload(await handle_mcp_proxy_tool("search_tools", {"query": "add"}, auth)) + tool_id = next(hit["tool_id"] for hit in hits if hit["name"] == "math_stdio-add") + result = await handle_mcp_proxy_tool("call_tool", {"tool_id": tool_id, "arguments": arguments}, auth) + assert result.isError is True + assert result.content[0].text == "arguments must be an object" + + asyncio.run_coroutine_threadsafe(check(), _proxy_server.loop).result(timeout=30) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py index 413785529d5..29711f80deb 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py @@ -1,355 +1,50 @@ -import json -from collections.abc import Iterator -from unittest.mock import AsyncMock, patch - import pytest -from mcp.types import CallToolResult, TextContent, Tool +from mcp.shared.exceptions import McpError +from pydantic import AnyUrl from litellm.proxy._experimental.mcp_server import server -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing -from litellm.proxy._experimental.mcp_server.tool_search import ( - MCP_PROXY_CALL_TOOL_NAME, - MCP_PROXY_SCHEMA_TOOL_NAME, - MCP_PROXY_SEARCH_TOOL_NAME, - handle_mcp_proxy_tool, - mcp_proxy_tool_id, - with_mcp_proxy_identity, -) -from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth -from litellm.types.mcp import MCPTransport -from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode +from litellm.proxy._types import UserAPIKeyAuth -TOOL = Tool.model_validate( - { - "name": "math_stdio-add", - "description": "Add two numbers", - "inputSchema": { - "type": "object", - "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}, - "required": ["a", "b"], - }, - "outputSchema": {"type": "object"}, - "_meta": {"litellm.ai/proxy_tool_identity": {"server_id": "server-1", "tool_name": "math_stdio-add"}}, - } -) AUTH = UserAPIKeyAuth(api_key="key") -def _text(result: CallToolResult) -> object: - return json.loads(result.content[0].text) - - -@pytest.mark.asyncio -async def test_proxy_search_returns_opaque_id_and_schema() -> None: - with ( - patch( # test-quality-ok: isolate authorized catalog owner - "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", - new_callable=AsyncMock, - return_value=AggregateToolListing(tools=[TOOL], outcomes={}), - ), - ): - result = await handle_mcp_proxy_tool(MCP_PROXY_SEARCH_TOOL_NAME, {"query": "add"}, AUTH) - - item = _text(result)[0] - assert item["tool_id"] == mcp_proxy_tool_id(TOOL) - assert item["name"] == TOOL.name - assert "inputSchema" not in item - assert "outputSchema" not in item - assert len(item["tool_id"]) == 32 - - -@pytest.mark.asyncio -async def test_proxy_schema_and_call_resolve_current_authorized_catalog() -> None: - executed = CallToolResult(content=[TextContent(type="text", text="3")], isError=False) - with ( - patch( # test-quality-ok: isolate authorized catalog owner - "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", - new_callable=AsyncMock, - return_value=AggregateToolListing(tools=[TOOL], outcomes={}), - ), - patch( # test-quality-ok: isolate execution delegate seam - "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", - new_callable=AsyncMock, - return_value=executed, - ) as call, - ): - schema = await handle_mcp_proxy_tool( - MCP_PROXY_SCHEMA_TOOL_NAME, - {"tool_id": mcp_proxy_tool_id(TOOL)}, - AUTH, - ) - result = await handle_mcp_proxy_tool( - MCP_PROXY_CALL_TOOL_NAME, - {"tool_id": mcp_proxy_tool_id(TOOL), "arguments": {"a": 1, "b": 2}}, - AUTH, - ) - - assert _text(schema)["inputSchema"] == TOOL.inputSchema - assert result is executed - assert call.await_args.kwargs["tool_name"] == TOOL.name - assert call.await_args.kwargs["arguments"] == {"a": 1, "b": 2} - assert call.await_args.kwargs["requested_server_id"] == "server-1" - - -@pytest.mark.asyncio -async def test_proxy_rejects_stale_id_and_invalid_arguments_before_dispatch() -> None: - with ( - patch( # test-quality-ok: isolate authorized catalog owner - "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", - new_callable=AsyncMock, - return_value=AggregateToolListing(tools=[TOOL], outcomes={}), - ), - patch( # test-quality-ok: isolate execution delegate seam - "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", new_callable=AsyncMock - ) as call, - ): - stale = await handle_mcp_proxy_tool(MCP_PROXY_SCHEMA_TOOL_NAME, {"tool_id": "stale"}, AUTH) - invalid = await handle_mcp_proxy_tool( - MCP_PROXY_CALL_TOOL_NAME, - {"tool_id": mcp_proxy_tool_id(TOOL), "arguments": "wrong"}, - AUTH, - ) - falsy = await handle_mcp_proxy_tool( - MCP_PROXY_CALL_TOOL_NAME, - {"tool_id": mcp_proxy_tool_id(TOOL), "arguments": False}, - AUTH, - ) - invalid_schema = await handle_mcp_proxy_tool( - MCP_PROXY_CALL_TOOL_NAME, - {"tool_id": mcp_proxy_tool_id(TOOL), "arguments": {"a": "wrong"}}, - AUTH, - ) - - assert stale.isError is True - assert invalid.isError is True - assert falsy.isError is True - assert invalid_schema.isError is True - call.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_proxy_call_builds_logging_object() -> None: - from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode - - sentinel = object() - result = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) +@pytest.fixture +def proxy_mode(): token = _mcp_proxy_mode.set(True) try: - with ( - patch.object( # test-quality-ok: isolate logging pipeline seam - server, "_build_virtual_call_logging_obj", new_callable=AsyncMock, return_value=sentinel - ) as build, - patch( # test-quality-ok: isolate proxy dispatch seam - "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_proxy_tool", - new_callable=AsyncMock, - return_value=result, - ) as handle, - ): - actual = await server._dispatch_virtual_mcp_tool( - name=MCP_PROXY_CALL_TOOL_NAME, - arguments={"tool_id": "id", "arguments": {}}, - user_api_key_auth=AUTH, - client_ip=None, - ) + yield finally: _mcp_proxy_mode.reset(token) - assert actual is result - build.assert_awaited_once() - assert handle.await_args.kwargs["litellm_logging_obj"] is sentinel - @pytest.mark.asyncio +@pytest.mark.usefixtures("proxy_mode") async def test_proxy_call_rejects_non_proxy_tool_names() -> None: - from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode - - token = _mcp_proxy_mode.set(True) - try: - result = await server._dispatch_virtual_mcp_tool( - name="math_stdio-add", - arguments={"a": 1, "b": 2}, - user_api_key_auth=AUTH, - client_ip=None, - ) - finally: - _mcp_proxy_mode.reset(token) + result = await server._dispatch_virtual_mcp_tool( + name="math_stdio-add", arguments={"a": 1, "b": 2}, user_api_key_auth=AUTH, client_ip=None + ) assert result is not None assert result.isError is True - assert "unavailable" in result.content[0].text + assert "unavailable on /mcp/proxy" in result.content[0].text @pytest.mark.asyncio +@pytest.mark.usefixtures("proxy_mode") async def test_proxy_rejects_non_tool_protocol_operations() -> None: - from mcp.shared.exceptions import McpError - from pydantic import AnyUrl - - from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode - - token = _mcp_proxy_mode.set(True) - try: - with pytest.raises(McpError): - await server.list_prompts() - with pytest.raises(McpError): - await server.get_prompt("prompt", {}) - with pytest.raises(McpError): - await server.list_resources() - with pytest.raises(McpError): - await server.list_resource_templates() - with pytest.raises(McpError): - await server.read_resource(AnyUrl("https://example.com/resource")) - finally: - _mcp_proxy_mode.reset(token) - - -@pytest.mark.asyncio -async def test_proxy_list_mode_has_fixed_definitions_without_search_flag() -> None: - from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode - - token = _mcp_proxy_mode.set(True) - try: - with patch( # test-quality-ok: isolate authenticated MCP context seam - "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", - new_callable=AsyncMock, - return_value=(AUTH, None, None, None, None, None, None), - ): - tools = await server.handle_list_tools() - options = server.server.create_initialization_options() - finally: - _mcp_proxy_mode.reset(token) - - assert {tool.name for tool in tools} == { - MCP_PROXY_SEARCH_TOOL_NAME, - MCP_PROXY_SCHEMA_TOOL_NAME, - MCP_PROXY_CALL_TOOL_NAME, - } + options = server.server.create_initialization_options() assert options.capabilities.prompts is None assert options.capabilities.resources is None assert options.capabilities.tools is not None - -def _server(server_id: str, name: str, **overrides: object) -> MCPServer: - return MCPServer( - server_id=server_id, - name=name, - server_name=name, - url=f"http://{name}.test", - transport=MCPTransport.http, - **overrides, - ) - - -def _upstream_tool(prefix: str, name: str) -> Tool: - return Tool( - name=f"{prefix}-{name}", - description=f"{name} numbers", - inputSchema={"type": "object", "properties": {"a": {"type": "integer"}}, "required": ["a"]}, - ) - - -def _auth(**object_permission: object) -> UserAPIKeyAuth: - return UserAPIKeyAuth( - api_key="sk-scope", - object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="scope", **object_permission), - ) - - -def _ids(result: CallToolResult) -> dict[str, str]: - return {item["name"]: item["tool_id"] for item in _text(result)} - - -class TestMcpProxyAuthorizationScope: - """The real catalog resolver runs (server grants, tool grants, scope header, sentinel); only the - upstream tools/list fetch and the final upstream dispatch are faked.""" - - ALPHA = _server("srv-alpha", "alpha", tool_name_to_display_name={"add": "Add Numbers"}) - BETA = _server("srv-beta", "beta") - ALPHA_ADD = with_mcp_proxy_identity(_upstream_tool("alpha", "add"), "srv-alpha") - ALPHA_MULTIPLY = with_mcp_proxy_identity(_upstream_tool("alpha", "multiply"), "srv-alpha") - BETA_ADD = with_mcp_proxy_identity(_upstream_tool("beta", "add"), "srv-beta") - - @pytest.fixture - def rig(self) -> Iterator[AsyncMock]: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - - upstream = { - "srv-alpha": [_upstream_tool("alpha", "add"), _upstream_tool("alpha", "multiply")], - "srv-beta": [_upstream_tool("beta", "add")], - } - - async def fetch(server: MCPServer, **_: object) -> list[Tool]: - return list(upstream[server.server_id]) - - dispatched = AsyncMock( - return_value=CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) - ) - global_mcp_server_manager.registry.update({"srv-alpha": self.ALPHA, "srv-beta": self.BETA}) - with ( - patch.object( # test-quality-ok: the upstream MCP server is the only faked collaborator - global_mcp_server_manager, "_get_tools_from_server", new=AsyncMock(side_effect=fetch) - ), - patch.object(global_mcp_server_manager, "call_tool", new=dispatched), # test-quality-ok: dispatch seam - ): - yield dispatched - - async def _proxy( - self, name: str, arguments: dict[str, object], auth: UserAPIKeyAuth, **kwargs: object - ) -> CallToolResult: - return await handle_mcp_proxy_tool(name, arguments, auth, **kwargs) - - @pytest.mark.asyncio - async def test_search_and_schema_are_bounded_by_the_key_server_grant(self, rig: AsyncMock) -> None: - granted = _auth(mcp_servers=["srv-alpha"]) - - assert _ids(await self._proxy(MCP_PROXY_SEARCH_TOOL_NAME, {"query": "numbers"}, granted)) == { - "alpha-add": mcp_proxy_tool_id(self.ALPHA_ADD), - "alpha-multiply": mcp_proxy_tool_id(self.ALPHA_MULTIPLY), - } - denied_schema = await self._proxy( - MCP_PROXY_SCHEMA_TOOL_NAME, {"tool_id": mcp_proxy_tool_id(self.BETA_ADD)}, granted - ) - denied_call = await self._proxy( - MCP_PROXY_CALL_TOOL_NAME, {"tool_id": mcp_proxy_tool_id(self.BETA_ADD), "arguments": {"a": 1}}, granted - ) - assert denied_schema.isError is True and denied_call.isError is True - rig.assert_not_awaited() - - @pytest.mark.asyncio - async def test_no_mcp_servers_sentinel_hides_every_tool(self, rig: AsyncMock) -> None: - result = await self._proxy( - MCP_PROXY_SEARCH_TOOL_NAME, {"query": "numbers"}, _auth(mcp_servers=["no-mcp-servers"]) - ) - assert _text(result) == [] - rig.assert_not_awaited() - - @pytest.mark.asyncio - async def test_tool_grant_hides_ungranted_tools_and_blocks_their_ids(self, rig: AsyncMock) -> None: - scoped = _auth(mcp_servers=["srv-alpha"], mcp_tool_permissions={"srv-alpha": ["add"]}) - - assert set(_ids(await self._proxy(MCP_PROXY_SEARCH_TOOL_NAME, {"query": "numbers"}, scoped))) == {"alpha-add"} - blocked = await self._proxy( - MCP_PROXY_CALL_TOOL_NAME, {"tool_id": mcp_proxy_tool_id(self.ALPHA_MULTIPLY), "arguments": {"a": 1}}, scoped - ) - assert blocked.isError is True - rig.assert_not_awaited() - - @pytest.mark.asyncio - async def test_same_named_tools_keep_distinct_ids_and_dispatch_to_their_own_server(self, rig: AsyncMock) -> None: - both = _auth(mcp_servers=["srv-alpha", "srv-beta"]) - - ids = _ids(await self._proxy(MCP_PROXY_SEARCH_TOOL_NAME, {"query": "add"}, both)) - assert set(ids) == {"alpha-add", "beta-add"}, "display-name overrides must not rename proxy identities" - assert ids["alpha-add"] != ids["beta-add"] - - result = await self._proxy(MCP_PROXY_CALL_TOOL_NAME, {"tool_id": ids["beta-add"], "arguments": {"a": 1}}, both) - assert result.isError is False - rig.assert_awaited_once() - assert rig.await_args.kwargs["server_name"] == "beta" - assert rig.await_args.kwargs["name"] == "add" - - @pytest.mark.asyncio - async def test_server_scope_header_narrows_search_within_the_grant(self, rig: AsyncMock) -> None: - both = _auth(mcp_servers=["srv-alpha", "srv-beta"]) - scoped = await self._proxy(MCP_PROXY_SEARCH_TOOL_NAME, {"query": "add"}, both, mcp_servers=["beta"]) - assert set(_ids(scoped)) == {"beta-add"} - rig.assert_not_awaited() + with pytest.raises(McpError): + await server.list_prompts() + with pytest.raises(McpError): + await server.get_prompt("prompt", {}) + with pytest.raises(McpError): + await server.list_resources() + with pytest.raises(McpError): + await server.list_resource_templates() + with pytest.raises(McpError): + await server.read_resource(AnyUrl("https://example.com/resource")) From 163847333242b7e75b1a15c27519beef2b1365ed Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 19:02:08 -0700 Subject: [PATCH 156/411] fix(bedrock): keep deletion response IDs in request context --- litellm/llms/bedrock/files/transformation.py | 18 ++++++------- .../test_bedrock_files_transformation.py | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 90b539ff37c..9875ac2b9c3 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -13,7 +13,7 @@ from urllib.parse import unquote import httpx from httpx import Headers, Response from openai.types.file_deleted import FileDeleted -from pydantic import BaseModel, ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter from typing_extensions import ReadOnly from litellm._logging import verbose_logger @@ -61,7 +61,11 @@ from ..base_aws_llm import BaseAWSLLM from ..common_utils import BedrockError, merge_bedrock_aws_request_params, resolve_s3_encryption_key_id S3_SIGNED_REQUEST_HEADERS_PARAM: Final = "_s3_signed_request_headers" -S3_DELETE_FILE_ID_PARAM: Final = "_s3_delete_file_id" + + +class _S3DeleteContext(BaseModel): + file_id: str = Field(min_length=1) + # litellm_params key carrying the size of the body uploaded to S3, handed from # `transform_create_file_request` to `transform_create_file_response`. @@ -1187,11 +1191,9 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): optional_params: Mapping[str, object], litellm_params: MutableMapping[str, object], ) -> tuple[str, dict[str, str]]: - request: Final = self._transform_s3_file_request( + return self._transform_s3_file_request( file_id=file_id, method="DELETE", optional_params=optional_params, litellm_params=litellm_params ) - litellm_params[S3_DELETE_FILE_ID_PARAM] = file_id - return request def transform_delete_file_response( self, @@ -1205,10 +1207,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): message=raw_response.text or f"S3 file deletion returned HTTP {raw_response.status_code}", headers=raw_response.headers, ) - file_id: Final = litellm_params.get(S3_DELETE_FILE_ID_PARAM) - if not isinstance(file_id, str) or not file_id: - raise ValueError("Missing file id for Bedrock file deletion response") - return FileDeleted(id=file_id, deleted=True, object="file") + context: Final = _S3DeleteContext.model_validate(logging_obj.model_call_details.get("additional_args")) + return FileDeleted(id=context.file_id, deleted=True, object="file") def transform_list_files_request( self, diff --git a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py index 2c02a58663e..3b01a4f2054 100644 --- a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py @@ -1861,6 +1861,33 @@ class TestBedrockFileDeletion: S3_URI: Final = "s3://my-bucket/litellm-bedrock-files-model-abc.jsonl" URL: Final = "https://s3.us-west-2.amazonaws.com/my-bucket/litellm-bedrock-files-model-abc.jsonl" + def test_interleaved_deletions_keep_their_own_file_ids(self, monkeypatch: pytest.MonkeyPatch) -> None: + import httpx + + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + monkeypatch.setenv("AWS_S3_BUCKET_NAME", "my-bucket") + config: Final = BedrockFilesConfig() + params: Final = { + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + } + file_ids: Final = (self.S3_URI, "s3://my-bucket/litellm-bedrock-files-model-second.jsonl") + for file_id in file_ids: + config.transform_delete_file_request(file_id=file_id, optional_params={}, litellm_params=params) + + deleted: Final = tuple( + config.transform_delete_file_response( + raw_response=httpx.Response(204), + logging_obj=MagicMock(model_call_details={"additional_args": {"file_id": file_id}}), + litellm_params=params, + ).id + for file_id in file_ids + ) + + assert deleted == file_ids + def test_delete_file_sends_signed_delete_and_returns_matching_id(self, monkeypatch: pytest.MonkeyPatch) -> None: import httpx import respx From 0086b62b4575fb5bb69653c2ad3d196822bd3f13 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 02:05:08 +0000 Subject: [PATCH 157/411] fix(cost-map): keep first fetch blocking, run retries in background Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 4 +- .../litellm_core_utils/get_model_cost_map.py | 132 +++++++++- litellm/proxy/proxy_server.py | 23 +- .../test_get_model_cost_map.py | 235 +++++++++++++----- 4 files changed, 296 insertions(+), 98 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 42c0ea881fd..4e3754399da 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -541,7 +541,7 @@ _key_management_system: Optional["KeyManagementSystem"] = None #### PII MASKING #### output_parse_pii: bool = False ############################################# -from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete model_cost = get_model_cost_map(url=model_cost_map_url) cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount @@ -2397,3 +2397,5 @@ def __getattr__(name: str) -> Any: # ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time + +mark_litellm_import_complete() diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 9cba5db8ab7..4118cd3420e 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -12,6 +12,7 @@ import asyncio import json import os import random +import threading import time from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -159,6 +160,15 @@ class GetModelCostMap: RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504}) MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3 MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0 +_litellm_import_complete = threading.Event() + + +def mark_litellm_import_complete() -> None: + _litellm_import_complete.set() + + +def _start_daemon_thread(fn: Callable[[], None]) -> None: + threading.Thread(target=fn, name="litellm-model-cost-map-retry", daemon=True).start() @dataclass(frozen=True, slots=True) @@ -297,9 +307,15 @@ def _fetch_remote_model_cost_map_with_retry_sync( sleep: Callable[[float], None], rng: random.Random, client: _SyncGetClient, + starting_attempt: int = 1, + initial_outcome: _FetchAttemptRetryable | None = None, ) -> ModelCostMapReloadResult: - for attempt in range(1, max_attempts + 1): - outcome = _attempt_fetch_sync(client=client, url=url, timeout=timeout) + for attempt in range(starting_attempt, max_attempts + 1): + outcome = ( + initial_outcome + if initial_outcome is not None and attempt == starting_attempt + else _attempt_fetch_sync(client=client, url=url, timeout=timeout) + ) if not isinstance(outcome, _FetchAttemptRetryable): return outcome wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng) @@ -464,6 +480,70 @@ def _finalize_model_cost_map(model_cost: dict) -> dict: return _expand_model_aliases(model_cost) +def adopt_model_cost_map( + new_model_cost_map: dict, # mutable-ok: public API preserves the mutable cost-map contract +) -> int: + import litellm + from litellm import utils + + litellm.model_cost = new_model_cost_map + utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation + litellm.add_known_models(model_cost_map=new_model_cost_map) + fetched_model_count: Final = len(new_model_cost_map) if new_model_cost_map else 0 + utils.reapply_runtime_model_cost_registrations() + return fetched_model_count + + +def _continue_remote_fetch_in_background( + url: str, + timeout: int, + max_attempts: int, + sleep: Callable[[float], None], + rng: random.Random, + client: _SyncGetClient, + first_outcome: _FetchAttemptRetryable, + apply: Callable[[dict], object], # mutable-ok: injected callback receives the mutable cost-map dict +) -> None: + try: + result: Final = _fetch_remote_model_cost_map_with_retry_sync( + url=url, + timeout=timeout, + max_attempts=max_attempts, + sleep=sleep, + rng=rng, + client=client, + initial_outcome=first_outcome, + ) + if isinstance(result, ModelCostMapReloadUnavailable): + verbose_logger.warning( + "LiteLLM: Failed to fetch remote model cost map from %s after %d attempts; keeping local backup", + url, + max_attempts, + ) + return + backup_model_count: Final = GetModelCostMap._get_backup_model_count() # pyright: ignore[reportPrivateUsage] # integrity cache + if not GetModelCostMap.validate_model_cost_map( + fetched_map=result.model_cost_map, + backup_model_count=backup_model_count, + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. Keeping local backup. url=%s", + url, + ) + _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" + return + finalized_map: Final = _finalize_model_cost_map(result.model_cost_map) + _litellm_import_complete.wait() + apply(finalized_map) + _cost_map_source_info.source = "remote" + _cost_map_source_info.url = url + _cost_map_source_info.is_env_forced = False + _cost_map_source_info.fallback_reason = None + _cost_map_source_info.loaded_at = datetime.now(timezone.utc) + except Exception as e: + verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e) + + def get_model_cost_map( url: str, timeout: int = 5, @@ -471,14 +551,19 @@ def get_model_cost_map( sleep: Callable[[float], None] = time.sleep, rng: random.Random | None = None, client: "_SyncGetClient | None" = None, + start_background: Callable[[Callable[[], None]], None] = _start_daemon_thread, + apply: Callable[ # mutable-ok: injected callback receives the mutable cost-map dict + [dict], + object, + ] = adopt_model_cost_map, ) -> dict: """ Public entry point — returns the model cost map dict. 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only. - 2. Otherwise fetches from ``url``, retrying transient HTTP errors - (429/5xx/transport) with Retry-After-aware backoff, validates - integrity, and falls back to the local backup on any failure. + 2. Otherwise fetches from ``url``, validates the first response, and falls + back to the local backup while retrying transient HTTP errors in the + background. Only the backup model count is cached (a single int) for validation. The full backup dict is only parsed when it must be *returned* as a @@ -497,14 +582,35 @@ def get_model_cost_map( _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False - result: Final = _fetch_remote_model_cost_map_with_retry_sync( - url=url, - timeout=timeout, - max_attempts=max_attempts, - sleep=sleep, - rng=rng if rng is not None else random.Random(), - client=client if client is not None else httpx, - ) + fetch_client: Final = client if client is not None else httpx + fetch_rng: Final = rng if rng is not None else random.Random() + first_outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout) + if isinstance(first_outcome, _FetchAttemptRetryable): + verbose_logger.warning( + "LiteLLM: model cost map fetch attempt 1/%d failed: %s; " + "using local backup while retrying in the background", + max_attempts, + first_outcome.reason, + ) + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = f"Remote fetch failed: {first_outcome.reason}" + local_map: Final = _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + if max_attempts > 1: + start_background( + lambda: _continue_remote_fetch_in_background( + url=url, + timeout=timeout, + max_attempts=max_attempts, + sleep=sleep, + rng=fetch_rng, + client=fetch_client, + first_outcome=first_outcome, + apply=apply, + ) + ) + return local_map + + result: Final = first_outcome if isinstance(result, ModelCostMapReloadUnavailable): verbose_logger.warning( "LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1cd08fe27c0..efe38376353 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -132,11 +132,7 @@ from litellm.types.utils import ( TextCompletionResponse, TokenCountResponse, ) -from litellm.utils import ( - _invalidate_model_cost_lowercase_map, - load_credentials_from_list, - reapply_runtime_model_cost_registrations, -) +from litellm.utils import load_credentials_from_list if TYPE_CHECKING: from aiohttp import ClientSession @@ -4411,20 +4407,9 @@ def resolve_classifier_plugin( def _swap_in_model_cost_map(new_model_cost_map: dict) -> int: - """Adopt a freshly fetched cost map into this process's litellm state, return the model count""" - litellm.model_cost = new_model_cost_map - # Invalidate case-insensitive lookup map since model_cost was replaced - _invalidate_model_cost_lowercase_map() - # Repopulate provider model sets (e.g. litellm.anthropic_models) so that - # wildcard patterns like "anthropic/*" include any newly added models. - litellm.add_known_models(model_cost_map=new_model_cost_map) - # Counted before the re-apply below, which writes into this same dict, so the - # number reported describes the fetched price data alone. - fetched_model_count: Final = len(new_model_cost_map) if new_model_cost_map else 0 - # The swap discards everything registered at runtime (deployment model_info, - # register_model overrides), so put it back on top of the fresh catalog. - reapply_runtime_model_cost_registrations() - return fetched_model_count + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + + return adopt_model_cost_map(new_model_cost_map) def should_load_db_object(object_type: str | SupportedDBObjectType) -> bool: diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 18185126775..dce1a431f13 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -20,21 +20,18 @@ from litellm.litellm_core_utils.get_model_cost_map import ( GetModelCostMap, _count_model_entries, _finalize_model_cost_map, + adopt_model_cost_map, ) def _load_root_cost_map() -> dict: - path = os.path.join( - os.path.dirname(__file__), "../../../model_prices_and_context_window.json" - ) + path = os.path.join(os.path.dirname(__file__), "../../../model_prices_and_context_window.json") with open(path) as f: return json.load(f) def _make_models(n: int) -> dict: - return { - f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n) - } + return {f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n)} def test_count_model_entries_excludes_reserved_keys(): @@ -117,9 +114,7 @@ def test_finalize_pops_key_and_installs_rules(): def test_finalize_with_no_block_clears_rules(): previous = list(get_fallback_generalization_rules()) try: - set_fallback_generalizations( - [{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}] - ) + set_fallback_generalizations([{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}]) _finalize_model_cost_map(_make_models(2)) assert match_capability_generalizations("x-1") is None finally: @@ -306,9 +301,7 @@ def test_get_model_cost_map_stamps_loaded_at(monkeypatch): from litellm.litellm_core_utils import get_model_cost_map as module monkeypatch.setattr(module._cost_map_source_info, "loaded_at", None) - client, _calls = _mock_client( - [httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client - ) + client, _calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) before = datetime.now(timezone.utc) module.get_model_cost_map(url="https://example.invalid/cost_map.json", client=client) @@ -317,6 +310,7 @@ def test_get_model_cost_map_stamps_loaded_at(monkeypatch): assert loaded_at is not None assert before <= loaded_at <= datetime.now(timezone.utc) + # --------------------------------------------------------------------------- # refetch_model_cost_map: retry/backoff behavior for runtime reloads # --------------------------------------------------------------------------- @@ -382,9 +376,7 @@ async def test_refetch_retries_429_honoring_retry_after(): ] ) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) assert len(result.model_cost_map) > 100 assert calls["count"] == 3 @@ -396,9 +388,7 @@ async def test_refetch_gives_up_after_max_attempts_with_exponential_backoff(): """All 429 without Retry-After: exponential backoff waits, then a failure value.""" client, calls = _mock_client([httpx.Response(429)]) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) assert "429" in result.reason assert "after 3 attempts" in result.reason @@ -418,9 +408,7 @@ async def test_refetch_caps_retry_after_wait(): ] ) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) assert sleeper.waits == [30.0] @@ -435,9 +423,7 @@ async def test_refetch_retries_transport_errors(): ] ) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) assert calls["count"] == 2 assert len(sleeper.waits) == 1 @@ -448,9 +434,7 @@ async def test_refetch_non_retryable_status_fails_immediately(): """A 404 is permanent: one attempt, no sleeps, failure value.""" client, calls = _mock_client([httpx.Response(404)]) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) assert "404" in result.reason assert calls["count"] == 1 @@ -461,9 +445,7 @@ async def test_refetch_non_retryable_status_fails_immediately(): async def test_refetch_invalid_json_fails_immediately(): client, calls = _mock_client([httpx.Response(200, content=b"not json")]) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) assert "invalid JSON" in result.reason assert calls["count"] == 1 @@ -475,9 +457,7 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in(): """A drastically shrunk upstream file is rejected instead of being adopted.""" tiny = json.dumps(_make_models(60)).encode() client, _calls = _mock_client([httpx.Response(200, content=tiny)]) - result = await refetch_model_cost_map( - url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) assert "integrity validation" in result.reason @@ -520,62 +500,187 @@ class _SyncSleepRecorder: self.waits.append(seconds) -def test_boot_load_retries_transient_failures_instead_of_falling_back(): - """A refused connection then a 503 at pod boot used to pin the process to the bundled - backup for its lifetime; both are transient and must be retried before giving up.""" +class _BackgroundRecorder: + def __init__(self): + self.callbacks = [] + + def __call__(self, callback): + self.callbacks.append(callback) + + +def test_boot_load_returns_local_map_and_schedules_transient_retry(): + client, calls = _mock_client( + [ + httpx.ConnectError("connection refused"), + ], + client_cls=httpx.Client, + ) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + + cost_map = get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + ) + assert calls["count"] == 1 + assert sleeper.waits == [] + assert len(background.callbacks) == 1 + assert len(cost_map) > 100 + source = get_model_cost_map_source_info() + assert source["source"] == "local" + assert source["fallback_reason"] is not None + + +def test_background_retry_adopts_valid_remote_map(): client, calls = _mock_client( [ httpx.ConnectError("connection refused"), - httpx.Response(503), httpx.Response(200, content=_real_map_bytes()), ], client_cls=httpx.Client, ) sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + applied = [] - cost_map = get_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) + get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + apply=applied.append, + ) + assert calls["count"] == 1 + assert sleeper.waits == [] + assert len(background.callbacks) == 1 + + background.callbacks[0]() + + assert calls["count"] == 2 + assert len(sleeper.waits) == 1 + assert 2.0 <= sleeper.waits[0] < 3.0 + assert len(applied) == 1 + assert applied[0].keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} + source = get_model_cost_map_source_info() + assert source["source"] == "remote" + assert source["fallback_reason"] is None + + +def test_background_retry_keeps_local_map_after_remaining_failures(): + client, calls = _mock_client( + [httpx.ConnectError("connection refused"), httpx.Response(503)], + client_cls=httpx.Client, + ) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + applied = [] + + get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + apply=applied.append, + ) + background.callbacks[0]() assert calls["count"] == 3 assert len(sleeper.waits) == 2 assert 2.0 <= sleeper.waits[0] < 3.0 assert 4.0 <= sleeper.waits[1] < 5.0 + assert applied == [] + assert get_model_cost_map_source_info()["source"] == "local" + + +def test_boot_load_does_not_schedule_non_retryable_failure(): + client, calls = _mock_client([httpx.Response(404)], client_cls=httpx.Client) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + + get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + ) + + assert calls["count"] == 1 + assert sleeper.waits == [] + assert background.callbacks == [] source = get_model_cost_map_source_info() - assert source["source"] == "remote" - assert source["fallback_reason"] is None + assert source["source"] == "local" + assert source["fallback_reason"] is not None + + +def test_boot_load_success_does_not_schedule_background_retry(): + client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + + cost_map = get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + ) + assert calls["count"] == 1 + assert sleeper.waits == [] + assert background.callbacks == [] + assert get_model_cost_map_source_info()["source"] == "remote" assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} -def test_boot_load_honors_retry_after_then_falls_back_after_max_attempts(): - """An outage longer than the retry budget still ends on the bundled backup, and the - recorded fallback reason says how many attempts were spent so operators can tell.""" - client, calls = _mock_client( - [httpx.Response(429, headers={"Retry-After": "7"})], client_cls=httpx.Client +def test_boot_load_with_one_attempt_does_not_schedule_background_retry(): + client, calls = _mock_client([httpx.ConnectError("connection refused")], client_cls=httpx.Client) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + + get_model_cost_map( + url=_URL, + max_attempts=1, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, ) - sleeper = _SyncSleepRecorder() - cost_map = get_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) - - assert calls["count"] == 3 - assert sleeper.waits == [7.0, 7.0] - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert "after 3 attempts" in source["fallback_reason"] - assert len(cost_map) > 100 - - -def test_boot_load_does_not_retry_permanent_failures(): - """A 404 or a malformed URL cannot heal by waiting: one attempt, no sleeps, backup.""" - client, calls = _mock_client([httpx.Response(404)], client_cls=httpx.Client) - sleeper = _SyncSleepRecorder() - - get_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert calls["count"] == 1 assert sleeper.waits == [] + assert background.callbacks == [] assert get_model_cost_map_source_info()["source"] == "local" - get_model_cost_map(url="not a url", sleep=sleeper, rng=random.Random(0)) - assert sleeper.waits == [] - assert get_model_cost_map_source_info()["source"] == "local" + +def test_adopt_model_cost_map_replays_runtime_registration_and_provider_models(): + import litellm + from litellm import utils as litellm_utils + + original_model_cost = litellm.model_cost + original_registry = dict(litellm_utils._runtime_registered_model_cost) + original_anthropic_models = set(litellm.anthropic_models) + try: + litellm.register_model( + model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}} + ) + + models_count = adopt_model_cost_map({"anthropic/new-model": {"litellm_provider": "anthropic", "mode": "chat"}}) + + assert models_count == 1 + assert "anthropic/new-model" in litellm.anthropic_models + assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321 + finally: + litellm.model_cost = original_model_cost # test-quality-ok: restore the module state changed by adoption + litellm_utils._runtime_registered_model_cost.clear() + litellm_utils._runtime_registered_model_cost.update(original_registry) + litellm.anthropic_models.clear() + litellm.anthropic_models.update(original_anthropic_models) + litellm_utils._invalidate_model_cost_lowercase_map() def test_boot_load_respects_local_env_override(monkeypatch): From ddedb4867b8cedea4ac2223dbc636f813e8bf997 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:09:44 -0700 Subject: [PATCH 158/411] fix: discard a streamed rewrite that drops or adds a tool call A guardrail that removes or adds a tool call on an ended stream used to be silently ignored: every handler substitutes the original list on a count mismatch and the executor skipped its observer once the translation could deliver rewrites. The executor now tracks the count change on the observer and releases the original chunks with the discard warning on every translation, matching what the merge base did for any tool call rewrite --- .../proxy/policy_engine/pipeline_executor.py | 20 ++++-- .../policy_engine/test_pipeline_executor.py | 23 +++++- .../proxy_logging/test_guardrail_pipeline.py | 71 +++++++++++++++++++ 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index c870bd7d2ec..a51468cc0fb 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -78,6 +78,10 @@ def _rewrote(sent: tuple[object, ...] | None, returned: tuple[object, ...] | Non return sent is not None and returned is not None and returned != sent +def _changed_count(sent: tuple[object, ...] | None, returned: tuple[object, ...] | None) -> bool: + return sent is not None and returned is not None and len(returned) != len(sent) + + _GuardrailMethodT = TypeVar("_GuardrailMethodT", bound=Callable[..., object]) @@ -91,8 +95,9 @@ class _StreamRewriteObserver(CustomGuardrail): guardrail. It records whether the guardrail returned different output than it was given, which for guardrails like Bedrock's ANONYMIZED action is only known at runtime. Text and tool-call rewrites are deliverable on translations that write them back across the - buffered chunks (``delivers_ended_stream_rewrites``); rewrites on any other translation - are discarded by the executor, which releases the original chunks. + buffered chunks (``delivers_ended_stream_rewrites``); rewrites on any other translation, + and a rewrite that drops or adds a tool call on any translation, are discarded by the + executor, which releases the original chunks. The inner guardrail's ``apply_guardrail`` already records the guardrail information and span, so the observer's stays out of ``log_guardrail_information``.""" @@ -101,6 +106,7 @@ class _StreamRewriteObserver(CustomGuardrail): self.inner: Final = inner self.rewrote_texts = False self.rewrote_tool_calls = False + self.changed_tool_call_count = False def structured_messages_cover_full_request(self) -> bool: return self.inner.structured_messages_cover_full_request() @@ -118,9 +124,11 @@ class _StreamRewriteObserver(CustomGuardrail): outputs: Final = await self.inner.apply_guardrail( inputs=inputs, request_data=request_data, input_type=input_type, logging_obj=logging_obj ) + returned_tool_shapes: Final = _tool_call_shapes(outputs.get("tool_calls")) self.rewrote_texts = self.rewrote_texts or _rewrote(sent_texts, _text_snapshot(outputs.get("texts"))) - self.rewrote_tool_calls = self.rewrote_tool_calls or _rewrote( - sent_tool_shapes, _tool_call_shapes(outputs.get("tool_calls")) + self.rewrote_tool_calls = self.rewrote_tool_calls or _rewrote(sent_tool_shapes, returned_tool_shapes) + self.changed_tool_call_count = self.changed_tool_call_count or _changed_count( + sent_tool_shapes, returned_tool_shapes ) return outputs @@ -325,7 +333,9 @@ class PipelineExecutor: except UndeliverableStreamRewrite: _release_original_chunks(step.guardrail, streaming_chunks, originals) else: - if not deliver_rewrites and (observer.rewrote_texts or observer.rewrote_tool_calls): + if observer.changed_tool_call_count or ( + not deliver_rewrites and (observer.rewrote_texts or observer.rewrote_tool_calls) + ): _release_original_chunks(step.guardrail, streaming_chunks, originals) if not callback.records_own_guardrail_information: add_guardrail_to_applied_guardrails_header(request_data=hook_input, guardrail_name=step.guardrail) diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 5d042d3c1ad..16401ccdaad 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1106,7 +1106,8 @@ class _WritingTranslation: logging_obj=litellm_logging_obj, ) responses_so_far[0]["text"] = outputs["texts"][0] - responses_so_far[0]["tool_call"] = outputs["tool_calls"][0] + if len(outputs["tool_calls"]) == 1: + responses_so_far[0]["tool_call"] = outputs["tool_calls"][0] return responses_so_far @@ -1242,6 +1243,26 @@ async def test_streaming_step_delivers_tool_call_rewrite_through_writing_transla assert not any("discarded" in record.getMessage() for record in caplog.records) +class _ToolCallDroppingGuardrail(CustomGuardrail): + def __init__(self): + super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True) + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return {**inputs, "texts": ["hello [MASKED]"], "tool_calls": []} + + +@pytest.mark.asyncio +async def test_streaming_step_discards_whole_rewrite_when_guardrail_drops_a_tool_call(monkeypatch, caplog): + monkeypatch.setattr(litellm, "callbacks", [_ToolCallDroppingGuardrail()]) + chunks = [_chunk()] + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await _run_streaming_step(_WritingTranslation(), chunks) + + _assert_passed_with_discard_warning(result, caplog) + assert chunks == [_chunk()] + + @pytest.mark.asyncio async def test_streaming_step_discards_tool_call_rewrite_when_translation_lacks_write_back(monkeypatch, caplog): monkeypatch.setattr(litellm, "callbacks", [_TextAndToolCallRewritingGuardrail(rewrite_tool_call=True)]) diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 3b59bbac3fd..d46881d045b 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -2322,3 +2322,74 @@ async def test_streaming_iterator_hook_pipeline_delivers_function_call_rewrite_o assert delivered[4]["item"]["arguments"] == '{"fruit": "[MASKED]"}' assert delivered[5]["response"]["output"][0]["arguments"] == '{"fruit": "[MASKED]"}' assert "persimmon" not in json.dumps(delivered) + + +def _drop_tool_calls(inputs: Dict[str, Any]) -> Dict[str, Any]: + return {"tool_calls": []} + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_discards_dropped_tool_call_on_chat_chunks( + proxy_logging, make_user_api_key_auth, monkeypatch, caplog +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_drop_tool_calls)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"), + response=_async_chunk_iter(_tool_call_stream_chunks()), + request_data=data, + ) + ] + + assert delivered[0].choices[0].delta.tool_calls[0].function.arguments == '{"ssn": "123"}' + assert delivered[1].choices[0].finish_reason == "tool_calls" + assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog)) + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_discards_dropped_tool_call_on_anthropic_sse( + proxy_logging, make_user_api_key_auth, monkeypatch, caplog +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_drop_tool_calls)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"), + response=_async_chunk_iter(_anthropic_tool_use_sse_chunks()), + request_data=data, + ) + ] + + assert delivered == _anthropic_tool_use_sse_chunks() + assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog)) + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_pipeline_discards_dropped_tool_call_on_responses_events( + proxy_logging, make_user_api_key_auth, monkeypatch, caplog +): + monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(_drop_tool_calls)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/responses"), + response=_async_chunk_iter(_responses_function_call_events()), + request_data=data, + ) + ] + + assert delivered == _responses_function_call_events() + assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog)) From aa0a9ab3ea4dff53a22cbc60fbc0195c4ab6098c Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 02:29:56 +0000 Subject: [PATCH 159/411] refactor(cost-map): drop initial_outcome flag from retry loop Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/get_model_cost_map.py | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 90ff696f926..d9b5a492539 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -324,19 +324,14 @@ async def _fetch_remote_model_cost_map_with_retry( def _fetch_remote_model_cost_map_with_retry_sync( url: str, timeout: int, - max_attempts: int, + attempts: range, sleep: Callable[[float], None], rng: random.Random, client: _SyncGetClient, - starting_attempt: int = 1, - initial_outcome: _FetchAttemptRetryable | None = None, ) -> ModelCostMapReloadResult: - for attempt in range(starting_attempt, max_attempts + 1): - outcome = ( - initial_outcome - if initial_outcome is not None and attempt == starting_attempt - else _attempt_fetch_sync(client=client, url=url, timeout=timeout) - ) + max_attempts: Final = attempts.stop - 1 + for attempt in attempts: + outcome = _attempt_fetch_sync(client=client, url=url, timeout=timeout) if not isinstance(outcome, _FetchAttemptRetryable): return outcome wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng) @@ -557,18 +552,18 @@ def _continue_remote_fetch_in_background( sleep: Callable[[float], None], rng: random.Random, client: _SyncGetClient, - first_outcome: _FetchAttemptRetryable, + first_wait: float, apply: Callable[[dict], object], # mutable-ok: injected callback receives the mutable cost-map dict ) -> None: try: + sleep(first_wait) result: Final = _fetch_remote_model_cost_map_with_retry_sync( url=url, timeout=timeout, - max_attempts=max_attempts, + attempts=range(2, max_attempts + 1), sleep=sleep, rng=rng, client=client, - initial_outcome=first_outcome, ) if isinstance(result, ModelCostMapReloadUnavailable): verbose_logger.warning( @@ -652,19 +647,26 @@ def get_model_cost_map( local_map: Final = _finalize_loaded_model_cost_map( GetModelCostMap.load_local_model_cost_map_with_revision() ).model_cost_map - if max_attempts > 1: - start_background( - lambda: _continue_remote_fetch_in_background( - url=url, - timeout=timeout, - max_attempts=max_attempts, - sleep=sleep, - rng=fetch_rng, - client=fetch_client, - first_outcome=first_outcome, - apply=apply, - ) + first_wait: Final = _next_retry_wait( + outcome=first_outcome, + attempt=1, + max_attempts=max_attempts, + rng=fetch_rng, + ) + if isinstance(first_wait, ModelCostMapReloadUnavailable): + return local_map + start_background( + lambda: _continue_remote_fetch_in_background( + url=url, + timeout=timeout, + max_attempts=max_attempts, + sleep=sleep, + rng=fetch_rng, + client=fetch_client, + first_wait=first_wait, + apply=apply, ) + ) return local_map result: Final = first_outcome From 536a85b42967fd0c9c6b49d2df9298232034427a Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 02:48:17 +0000 Subject: [PATCH 160/411] fix(cost-map): keep register_model url fetch to a single attempt Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/utils.py | 2 +- tests/test_litellm/test_utils.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 36b48d3b8d8..8df28870544 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3079,7 +3079,7 @@ def register_model( # Convert stringified numbers to appropriate numeric types loaded_model_cost = model_cost elif isinstance(model_cost, str): - loaded_model_cost = litellm.get_model_cost_map(url=model_cost) + loaded_model_cost = litellm.get_model_cost_map(url=model_cost, max_attempts=1) if persist_across_reloads: _registrations: Final[Mapping[str, Mapping[str, object]]] = loaded_model_cost diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index fee5e3a2e4c..e90372141bd 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2,10 +2,12 @@ import asyncio import json import logging import os +import threading from datetime import datetime, timedelta, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import respx from jsonschema import validate @@ -2382,6 +2384,27 @@ def test_register_model_with_scientific_notation(): _invalidate_model_cost_lowercase_map() +@respx.mock +def test_register_model_url_fetch_uses_single_attempt(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + before = dict(litellm.model_cost) + threads_before = {thread.name for thread in threading.enumerate()} + route = respx.get("https://example.invalid/custom_pricing.json").mock( + return_value=httpx.Response(503) + ) + + litellm.register_model(model_cost="https://example.invalid/custom_pricing.json") + + threads_after = {thread.name for thread in threading.enumerate()} + assert route.call_count == 1 + assert not (threads_after - threads_before) & {"litellm-model-cost-map-retry"} + assert not any( + thread.name == "litellm-model-cost-map-retry" and thread.is_alive() + for thread in threading.enumerate() + ) + assert litellm.model_cost.keys() >= before.keys() + + def test_register_model_openrouter_without_slash(): """ Test that register_model handles openrouter models without '/' in the name. From 902dd7b2b6c61b282d3dafb6aee92607f53ecc81 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 8 Sep 2026 19:56:21 -0700 Subject: [PATCH 161/411] fix(mcp): log proxy tool dispatch exceptions (#40351) --- .../proxy/_experimental/mcp_server/server.py | 51 ++++++++++---- tests/mcp_tests/test_proxy_mcp_e2e.py | 30 ++++++++ .../mcp_server/test_mcp_proxy_mode.py | 70 ++++++++++++++++++- 3 files changed, 138 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 30793ef246d..9a52da0cab1 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -1017,18 +1017,45 @@ if MCP_AVAILABLE: if name == MCP_PROXY_CALL_TOOL_NAME else None ) - proxy_result: Final = await handle_mcp_proxy_tool( - name=name, - arguments=arguments or {}, # mutable-ok: proxy handler payload - user_api_key_dict=user_api_key_auth, - client_ip=client_ip, - mcp_servers=mcp_servers, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - litellm_logging_obj=proxy_logging_obj, - ) + try: + proxy_result: Final = await handle_mcp_proxy_tool( + name=name, + arguments=arguments or {}, # mutable-ok: proxy handler payload + user_api_key_dict=user_api_key_auth, + client_ip=client_ip, + mcp_servers=mcp_servers, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_logging_obj=proxy_logging_obj, + ) + except Exception as exc: + if proxy_logging_obj is not None: + from litellm.proxy.proxy_server import proxy_logging_obj as request_logging_obj + + failure_end: Final = datetime.now() # noqa: DTZ005 # matches the logging pipeline start time + failure_traceback: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) + try: + proxy_logging_obj.failure_handler(exc, failure_traceback, proxy_call_start, failure_end) + await proxy_logging_obj.async_failure_handler( + exc, failure_traceback, proxy_call_start, failure_end + ) + if not isinstance(exc, MCPUpstreamAuthError): + await request_logging_obj.post_call_failure_hook( + request_data={ # mutable-ok: failure hook mutates its request payload + "name": name, + "arguments": arguments, + "litellm_logging_obj": proxy_logging_obj, + }, + original_exception=exc, + user_api_key_dict=user_api_key_auth, + route="/mcp/call_tool", + traceback_str=failure_traceback, + ) + except Exception: + verbose_logger.exception("Error logging failed MCP proxy tool call") + raise if proxy_logging_obj is not None: return await _fire_mcp_tool_call_logging( logging_obj=proxy_logging_obj, diff --git a/tests/mcp_tests/test_proxy_mcp_e2e.py b/tests/mcp_tests/test_proxy_mcp_e2e.py index 5e5b9db2dc6..f0fc3d892e2 100644 --- a/tests/mcp_tests/test_proxy_mcp_e2e.py +++ b/tests/mcp_tests/test_proxy_mcp_e2e.py @@ -471,6 +471,7 @@ class ProxyCallRecorder(CustomLogger): def __init__(self) -> None: super().__init__() self.events: queue.Queue[str] = queue.Queue() + self.failures: queue.Queue[str] = queue.Queue() async def async_log_success_event( self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime @@ -479,6 +480,13 @@ class ProxyCallRecorder(CustomLogger): if isinstance(payload, dict) and payload.get("call_type") == "call_mcp_tool": self.events.put(json.dumps(payload, default=str)) + async def async_log_failure_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + payload = kwargs.get("standard_logging_object") + if isinstance(payload, dict) and payload.get("call_type") == "call_mcp_tool": + self.failures.put(json.dumps(payload, default=str)) + proxy_call_recorder = ProxyCallRecorder() @@ -636,6 +644,28 @@ class TestProxyMcpAuthorizationScope: assert payload["metadata"]["mcp_tool_call_metadata"]["name"] == "add" assert payload["metadata"]["mcp_tool_call_metadata"]["namespaced_tool_name"] == "math_restricted/add" + @pytest.mark.asyncio + async def test_proxy_scope_exception_returns_iserror_and_emits_failure_log(self, proxy_server_url: str) -> None: + async with _scoped_session( + proxy_server_url, + "sk-none", + **{"x-mcp-servers": "math_restricted", "x-litellm-call-id": "proxy-scope-denial"}, + ) as session: + result = await session.call_tool("call_tool", {"tool_id": "denied-scope", "arguments": {}}) + assert result.isError is True + assert result.content[0].text == ( + "Error: The key is not allowed to access the requested MCP servers: math_restricted" + ) + async with asyncio.timeout(10): + while True: + payload = json.loads(await asyncio.to_thread(proxy_call_recorder.failures.get, True, 5)) + if payload["id"] == "proxy-scope-denial": + break + assert payload["call_type"] == "call_mcp_tool" + assert payload["status"] == "failure" + assert payload["response_cost"] == 0 + assert "math_restricted" in payload["error_str"] + @pytest.mark.parametrize("arguments", ["wrong", False, None, [], 0]) def test_handler_rejects_non_object_arguments( self, proxy_server_url: str, _proxy_server: ProxyRig, arguments: object diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py index 29711f80deb..67b7c5a3414 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py @@ -1,10 +1,16 @@ +import json +from datetime import datetime + import pytest +from fastapi import HTTPException from mcp.shared.exceptions import McpError from pydantic import AnyUrl +import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._experimental.mcp_server import server from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth AUTH = UserAPIKeyAuth(api_key="key") @@ -48,3 +54,65 @@ async def test_proxy_rejects_non_tool_protocol_operations() -> None: await server.list_resource_templates() with pytest.raises(McpError): await server.read_resource(AnyUrl("https://example.com/resource")) + + +class FailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: list[tuple[str, str]] = [] + + async def async_log_failure_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.events.append(("failure", json.dumps(kwargs.get("standard_logging_object"), default=str))) + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.events.append(("success", json.dumps(kwargs.get("standard_logging_object"), default=str))) + + async def async_post_call_failure_hook( + self, + request_data: dict[str, object], + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> None: + self.events.append(("post_failure", json.dumps(request_data, default=str))) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("proxy_mode") +async def test_proxy_scope_exception_emits_failure_log(monkeypatch: pytest.MonkeyPatch) -> None: + recorder = FailureRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + auth = UserAPIKeyAuth( + api_key="scope-denial-key-hash", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="denied", mcp_servers=["no-mcp-servers"]), + ) + arguments = {"tool_id": "denied-scope", "arguments": {}} + + with pytest.raises(HTTPException) as denied: + await server._dispatch_virtual_mcp_tool( + name="call_tool", + arguments=arguments, + user_api_key_auth=auth, + client_ip=None, + mcp_servers=["ungranted"], + raw_headers={"authorization": "Bearer raw-scope-secret", "x-litellm-call-id": "scope-denial"}, + ) + + assert denied.value.status_code == 403 + assert denied.value.detail == {"error": "The key is not allowed to access the requested MCP servers: ungranted"} + assert [kind for kind, _ in recorder.events] == ["failure", "post_failure"] + payload = json.loads(recorder.events[0][1]) + assert payload["id"] == "scope-denial" + assert payload["call_type"] == "call_mcp_tool" + assert payload["status"] == "failure" + assert payload["response_cost"] == 0 + assert "ungranted" in payload["error_str"] + hook_payload = json.loads(recorder.events[1][1]) + assert hook_payload["standard_logging_object"] == payload + assert hook_payload["arguments"] == arguments + assert "raw_headers" not in hook_payload + assert "raw-scope-secret" not in recorder.events[1][1] From 4179f086e0196107dfc83da2a4e0bb1d15568edd Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 02:58:51 +0000 Subject: [PATCH 162/411] refactor(cost-map): share local-fallback and remote-accept paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/get_model_cost_map.py | 111 ++++++++---------- 1 file changed, 47 insertions(+), 64 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index d9b5a492539..67d06a0758e 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -531,6 +531,32 @@ def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMa return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) +def _use_local_backup(reason: str | None) -> dict: # mutable-ok: returns the mutable model-cost map contract + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = reason + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + + +def _accept_remote( + result: ModelCostMapReloaded, url: str +) -> dict | None: # mutable-ok: returns the mutable model-cost map contract + if not GetModelCostMap.validate_model_cost_map( + fetched_map=result.model_cost_map, + backup_model_count=GetModelCostMap._get_backup_model_count(), # pyright: ignore[reportPrivateUsage] # integrity cache + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", + url, + ) + return None + finalized: Final = _finalize_loaded_model_cost_map(result).model_cost_map + _cost_map_source_info.source = "remote" + _cost_map_source_info.url = url + _cost_map_source_info.is_env_forced = False + _cost_map_source_info.fallback_reason = None + return finalized + + def adopt_model_cost_map( new_model_cost_map: dict, # mutable-ok: public API preserves the mutable cost-map contract ) -> int: @@ -545,17 +571,20 @@ def adopt_model_cost_map( return fetched_model_count -def _continue_remote_fetch_in_background( +def _retry_remote_fetch_in_background( url: str, timeout: int, max_attempts: int, sleep: Callable[[float], None], rng: random.Random, client: _SyncGetClient, - first_wait: float, + first_outcome: _FetchAttemptRetryable, apply: Callable[[dict], object], # mutable-ok: injected callback receives the mutable cost-map dict ) -> None: try: + first_wait: Final = _next_retry_wait(outcome=first_outcome, attempt=1, max_attempts=max_attempts, rng=rng) + if isinstance(first_wait, ModelCostMapReloadUnavailable): + return sleep(first_wait) result: Final = _fetch_remote_model_cost_map_with_retry_sync( url=url, @@ -572,23 +601,12 @@ def _continue_remote_fetch_in_background( max_attempts, ) return - backup_model_count: Final = GetModelCostMap._get_backup_model_count() # pyright: ignore[reportPrivateUsage] # integrity cache - if not GetModelCostMap.validate_model_cost_map( - fetched_map=result.model_cost_map, - backup_model_count=backup_model_count, - ): - verbose_logger.warning( - "LiteLLM: Fetched model cost map failed integrity check. Keeping local backup. url=%s", - url, - ) + _litellm_import_complete.wait() + accepted: Final = _accept_remote(result, url) + if accepted is None: _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" return - _litellm_import_complete.wait() - apply(_finalize_loaded_model_cost_map(result).model_cost_map) - _cost_map_source_info.source = "remote" - _cost_map_source_info.url = url - _cost_map_source_info.is_env_forced = False - _cost_map_source_info.fallback_reason = None + apply(accepted) _cost_map_source_info.loaded_at = datetime.now(timezone.utc) except Exception as e: verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e) @@ -623,77 +641,42 @@ def get_model_cost_map( # Note: can't use get_secret_bool here — this runs during litellm.__init__ # before litellm._key_management_settings is set. if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": - _cost_map_source_info.source = "local" _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True - _cost_map_source_info.fallback_reason = None - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + return _use_local_backup(None) _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False fetch_client: Final = client if client is not None else httpx fetch_rng: Final = rng if rng is not None else random.Random() - first_outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout) - if isinstance(first_outcome, _FetchAttemptRetryable): + outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout) + if isinstance(outcome, ModelCostMapReloaded): + accepted: Final = _accept_remote(outcome, url) + return accepted if accepted is not None else _use_local_backup("Remote data failed integrity validation") + if isinstance(outcome, _FetchAttemptRetryable) and max_attempts > 1: verbose_logger.warning( "LiteLLM: model cost map fetch attempt 1/%d failed: %s; " "using local backup while retrying in the background", max_attempts, - first_outcome.reason, + outcome.reason, ) - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = f"Remote fetch failed: {first_outcome.reason}" - local_map: Final = _finalize_loaded_model_cost_map( - GetModelCostMap.load_local_model_cost_map_with_revision() - ).model_cost_map - first_wait: Final = _next_retry_wait( - outcome=first_outcome, - attempt=1, - max_attempts=max_attempts, - rng=fetch_rng, - ) - if isinstance(first_wait, ModelCostMapReloadUnavailable): - return local_map start_background( - lambda: _continue_remote_fetch_in_background( + lambda: _retry_remote_fetch_in_background( url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, rng=fetch_rng, client=fetch_client, - first_wait=first_wait, + first_outcome=outcome, apply=apply, ) ) - return local_map - - result: Final = first_outcome - if isinstance(result, ModelCostMapReloadUnavailable): + else: verbose_logger.warning( "LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.", url, - result.reason, + outcome.reason, ) - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = f"Remote fetch failed: {result.reason}" - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map - content: Final = result.model_cost_map - - # Validate using cached count (cheap int comparison, no file I/O) - if not GetModelCostMap.validate_model_cost_map( - fetched_map=content, - backup_model_count=GetModelCostMap._get_backup_model_count(), - ): - verbose_logger.warning( - "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", - url, - ) - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map - - _cost_map_source_info.source = "remote" - _cost_map_source_info.fallback_reason = None - return _finalize_loaded_model_cost_map(result).model_cost_map + return _use_local_backup(f"Remote fetch failed: {outcome.reason}") From 94f9230d13900897f764498169bdfa3c5fa2a4ea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:59:09 -0700 Subject: [PATCH 163/411] fix(proxy): type the new pipeline tests and keep tag values out of the deferral warning Every test this PR adds now annotates its fixture and parametrize parameters. The submit-time warning for a tag-matched deferred policy names only the policies, since a wildcard attachment pattern would let caller-provided tag text reach the log. --- litellm/proxy/utils.py | 6 +- .../policy_engine/test_response_retrieval.py | 42 ++-- .../proxy_logging/test_guardrail_pipeline.py | 216 ++++++++++-------- 3 files changed, 155 insertions(+), 109 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d144aa113d6..1604dea1003 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -620,19 +620,19 @@ def _defer_post_call_pipelines( "retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body " "does not govern the completed response: %s", response.id, - ", ".join(f"{policy_name} ({source})" for policy_name, source in tag_matched), + ", ".join(tag_matched), ) _withdraw_deferred_claims(data, deferred) def _tag_matched_deferrals( data: Mapping[str, object], deferred: Sequence[tuple[str, "GuardrailPipeline"]] -) -> tuple[tuple[str, str], ...]: +) -> tuple[str, ...]: sources: Final = _policy_state_metadata(data).get("policy_sources") if not isinstance(sources, dict): return () return tuple( - (policy_name, str(sources[policy_name])) + policy_name for policy_name, _pipeline in deferred if policy_name in sources and "tag:" in str(sources[policy_name]) ) diff --git a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py index 1fa1161191d..37849101b3a 100644 --- a/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py +++ b/tests/test_litellm/proxy/policy_engine/test_response_retrieval.py @@ -1,5 +1,5 @@ import logging -from collections.abc import Mapping +from collections.abc import Iterator, Mapping import pytest @@ -60,7 +60,7 @@ def _pipeline_policy(guardrail: str, mode: str = "post_call") -> dict[str, objec @pytest.fixture -def policy_engine(): +def policy_engine() -> Iterator[None]: policy_registry = get_policy_registry() attachment_registry = get_attachment_registry() policy_registry.load_policies( @@ -97,7 +97,7 @@ def _attached_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, str], .. ) -def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine): +def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine: None) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) @@ -111,7 +111,7 @@ def test_attaches_model_scoped_post_call_pipeline_to_retrieval(policy_engine): assert "guardrails" not in data["litellm_metadata"] -def test_key_and_team_context_also_governs_retrieval(policy_engine): +def test_key_and_team_context_also_governs_retrieval(policy_engine: None) -> None: data = _retrieval_data(UNGOVERNED_MODEL_ID) attach_post_call_pipelines_to_retrieval( @@ -121,7 +121,7 @@ def test_key_and_team_context_also_governs_retrieval(policy_engine): assert _attached_pipelines(data) == (("team-governance", "team-word-filter"),) -def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag(policy_engine): +def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag(policy_engine: None) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) @@ -129,7 +129,7 @@ def test_tag_attached_policy_is_not_re_matched_when_the_retrieval_carries_no_tag assert _attached_pipelines(data) == (("response-governance", "output-word-filter"),) -def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag(policy_engine): +def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag(policy_engine: None) -> None: data: dict[str, object] = { "response_id": _encoded_response_id(UNGOVERNED_MODEL_ID), "litellm_metadata": {"tags": ["governed"]}, @@ -141,7 +141,7 @@ def test_tag_attached_policy_governs_a_retrieval_whose_metadata_carries_the_tag( assert data["litellm_metadata"]["policy_sources"] == {"tag-governance": "tag:governed"} -def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine): +def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine: None) -> None: data = _retrieval_data(UNGOVERNED_MODEL_ID) attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=_router()) @@ -149,7 +149,7 @@ def test_retrieval_of_an_ungoverned_model_attaches_nothing(policy_engine): assert data == _retrieval_data(UNGOVERNED_MODEL_ID) -def test_already_attached_policy_is_not_attached_twice(policy_engine): +def test_already_attached_policy_is_not_attached_twice(policy_engine: None) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) router = _router() attach_post_call_pipelines_to_retrieval(data=data, user_api_key_dict=UserAPIKeyAuth(), llm_router=router) @@ -168,7 +168,9 @@ def _hidden_submit_model_warnings(caplog: pytest.LogCaptureFixture) -> list[str] ] -def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns(policy_engine, caplog): +def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns( + policy_engine: None, caplog: pytest.LogCaptureFixture +) -> None: data = _retrieval_data(WILDCARD_MODEL_ID) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): @@ -176,11 +178,14 @@ def test_wildcard_deployment_attaches_nothing_for_the_submitted_model_and_warns( assert data == _retrieval_data(WILDCARD_MODEL_ID) assert [ - "as model group openai/* (a wildcard deployment)" in message for message in _hidden_submit_model_warnings(caplog) + "as model group openai/* (a wildcard deployment)" in message + for message in _hidden_submit_model_warnings(caplog) ] == [True] -def test_aliased_model_group_still_attaches_its_own_policies_and_warns(policy_engine, caplog): +def test_aliased_model_group_still_attaches_its_own_policies_and_warns( + policy_engine: None, caplog: pytest.LogCaptureFixture +) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) router = _router({"gpt-mini": GOVERNED_MODEL_GROUP, "gpt-hidden": {"model": GOVERNED_MODEL_GROUP, "hidden": True}}) @@ -194,7 +199,9 @@ def test_aliased_model_group_still_attaches_its_own_policies_and_warns(policy_en ] == [True] -def test_plain_model_group_retrieval_does_not_warn_about_the_submitted_model(policy_engine, caplog): +def test_plain_model_group_retrieval_does_not_warn_about_the_submitted_model( + policy_engine: None, caplog: pytest.LogCaptureFixture +) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): attach_post_call_pipelines_to_retrieval( data=_retrieval_data(GOVERNED_MODEL_ID), @@ -209,7 +216,8 @@ def _ungoverned_retrieval_warnings(caplog: pytest.LogCaptureFixture) -> list[str return [ record.getMessage() for record in caplog.records - if record.levelno == logging.WARNING and "retrieved without its post_call policy pipelines" in record.getMessage() + if record.levelno == logging.WARNING + and "retrieved without its post_call policy pipelines" in record.getMessage() ] @@ -221,7 +229,9 @@ def _ungoverned_retrieval_warnings(caplog: pytest.LogCaptureFixture) -> list[str (None, "response id names no deployment"), ], ) -def test_unresolvable_response_id_attaches_nothing_and_warns(policy_engine, caplog, response_id, reason): +def test_unresolvable_response_id_attaches_nothing_and_warns( + policy_engine: None, caplog: pytest.LogCaptureFixture, response_id: str, reason: str +) -> None: data = {"response_id": response_id, "litellm_metadata": {}} with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): @@ -231,7 +241,7 @@ def test_unresolvable_response_id_attaches_nothing_and_warns(policy_engine, capl assert [message.endswith(f"({reason})") for message in _ungoverned_retrieval_warnings(caplog)] == [True] -def test_without_a_router_attaches_nothing_and_warns(policy_engine, caplog): +def test_without_a_router_attaches_nothing_and_warns(policy_engine: None, caplog: pytest.LogCaptureFixture) -> None: data = _retrieval_data(GOVERNED_MODEL_ID) with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): @@ -241,7 +251,7 @@ def test_without_a_router_attaches_nothing_and_warns(policy_engine, caplog): assert [message.endswith("(no router)") for message in _ungoverned_retrieval_warnings(caplog)] == [True] -def test_without_policy_engine_attaches_nothing_quietly(caplog): +def test_without_policy_engine_attaches_nothing_quietly(caplog: pytest.LogCaptureFixture) -> None: get_policy_registry().clear() data = _retrieval_data(GOVERNED_MODEL_ID) diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 63afce5d968..3bbdbbe2ccd 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -12,6 +12,7 @@ from __future__ import annotations import asyncio import json import logging +from collections.abc import Iterator from typing import Any, Callable, Dict, List from unittest.mock import AsyncMock, MagicMock, patch @@ -155,9 +156,7 @@ async def test_execute_guardrail_hook_unknown_hook_type_raises(proxy_logging, ma @pytest.mark.asyncio -async def test_execute_guardrail_with_load_balancing_routes_through_router( - proxy_logging, make_user_api_key_auth -): +async def test_execute_guardrail_with_load_balancing_routes_through_router(proxy_logging, make_user_api_key_auth): cb = _make_guardrail() router = MagicMock() router.get_available_guardrail = MagicMock(return_value={"callback": cb}) @@ -173,9 +172,7 @@ async def test_execute_guardrail_with_load_balancing_routes_through_router( @pytest.mark.asyncio -async def test_execute_guardrail_with_load_balancing_router_none_raises( - proxy_logging, make_user_api_key_auth -): +async def test_execute_guardrail_with_load_balancing_router_none_raises(proxy_logging, make_user_api_key_auth): with patch("litellm.proxy.proxy_server.llm_router", None): with pytest.raises(ValueError, match="Router not initialized"): await proxy_logging._execute_guardrail_with_load_balancing( @@ -188,9 +185,7 @@ async def test_execute_guardrail_with_load_balancing_router_none_raises( @pytest.mark.asyncio -async def test_execute_guardrail_with_load_balancing_no_callback_raises( - proxy_logging, make_user_api_key_auth -): +async def test_execute_guardrail_with_load_balancing_no_callback_raises(proxy_logging, make_user_api_key_auth): router = MagicMock() router.get_available_guardrail = MagicMock(return_value={"callback": None}) with patch("litellm.proxy.proxy_server.llm_router", router): @@ -210,9 +205,7 @@ async def test_execute_guardrail_with_load_balancing_no_callback_raises( @pytest.mark.asyncio -async def test_process_guardrail_callback_skipped_when_should_run_false( - proxy_logging, make_user_api_key_auth -): +async def test_process_guardrail_callback_skipped_when_should_run_false(proxy_logging, make_user_api_key_auth): cb = _make_guardrail() cb.should_run_guardrail = MagicMock(return_value=False) out = await proxy_logging._process_guardrail_callback( @@ -226,9 +219,7 @@ async def test_process_guardrail_callback_skipped_when_should_run_false( @pytest.mark.asyncio -async def test_process_guardrail_callback_returns_data_on_success( - proxy_logging, make_user_api_key_auth, monkeypatch -): +async def test_process_guardrail_callback_returns_data_on_success(proxy_logging, make_user_api_key_auth, monkeypatch): cb = _make_guardrail() cb.should_run_guardrail = MagicMock(return_value=True) proxy_logging._should_use_guardrail_load_balancing = MagicMock(return_value=False) @@ -343,14 +334,14 @@ async def test_maybe_execute_pipelines_no_pipelines_returns_data(proxy_logging, @pytest.mark.asyncio -async def test_maybe_execute_pipelines_skips_pipelines_with_other_mode(proxy_logging, make_user_api_key_auth, monkeypatch): +async def test_maybe_execute_pipelines_skips_pipelines_with_other_mode( + proxy_logging, make_user_api_key_auth, monkeypatch +): pipeline = MagicMock() pipeline.mode = "post_call" # not pre_call data = {"metadata": {"_guardrail_pipelines": [("p1", pipeline)]}, "model": "m", "messages": []} executed = MagicMock() - monkeypatch.setattr( - "litellm.proxy.policy_engine.pipeline_executor.PipelineExecutor.execute_steps", executed - ) + monkeypatch.setattr("litellm.proxy.policy_engine.pipeline_executor.PipelineExecutor.execute_steps", executed) out, replacement = await proxy_logging._maybe_execute_pipelines( data=data, user_api_key_dict=make_user_api_key_auth(), @@ -538,9 +529,7 @@ def test_handle_pipeline_result_block_enriches_with_guardrail_name_and_mode(): litellm.callbacks = [cb] try: with pytest.raises(HTTPException) as info: - ProxyLogging._handle_pipeline_result( - result=result, data={"model": "m"}, policy_name="p" - ) + ProxyLogging._handle_pipeline_result(result=result, data={"model": "m"}, policy_name="p") finally: litellm.callbacks = saved @@ -652,9 +641,7 @@ async def test_run_guardrail_with_metrics_records_error_and_enriches(monkeypatch monkeypatch.setattr(litellm, "callbacks", [prom]) with pytest.raises(HTTPException): - await ProxyLogging._run_guardrail_with_metrics( - callback=cb, coro=task(), hook_type="post_call" - ) + await ProxyLogging._run_guardrail_with_metrics(callback=cb, coro=task(), hook_type="post_call") assert detail["guardrail_name"] == "presidio" recorded = prom._record_guardrail_metrics.call_args.kwargs @@ -682,9 +669,7 @@ def _moderation_guardrail() -> MagicMock: @pytest.mark.asyncio -async def test_during_call_hook_records_latency_metric( - proxy_logging, make_user_api_key_auth, monkeypatch -): +async def test_during_call_hook_records_latency_metric(proxy_logging, make_user_api_key_auth, monkeypatch): cb = _moderation_guardrail() prom = _prometheus_callback() monkeypatch.setattr(litellm, "callbacks", [prom, cb]) @@ -703,9 +688,7 @@ async def test_during_call_hook_records_latency_metric( @pytest.mark.asyncio -async def test_post_call_success_hook_records_latency_metric( - proxy_logging, make_user_api_key_auth, monkeypatch -): +async def test_post_call_success_hook_records_latency_metric(proxy_logging, make_user_api_key_auth, monkeypatch): cb = _moderation_guardrail() prom = _prometheus_callback() monkeypatch.setattr(litellm, "callbacks", [prom, cb]) @@ -733,9 +716,7 @@ async def test_post_call_success_hook_records_latency_metric( async def test_process_prompt_template_no_op_when_no_prompt_spec(proxy_logging, monkeypatch): from litellm.proxy.prompts import prompt_registry - monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: None - ) + monkeypatch.setattr(prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: None) data: Dict[str, Any] = {"messages": [{"role": "user"}], "model": "m", "temperature": 0.1} await proxy_logging._process_prompt_template( data=data, @@ -760,9 +741,7 @@ async def test_process_prompt_template_applies_when_spec_resolves(proxy_logging, "get_prompt_callback_for_prompt", lambda *a, **kw: custom_logger, ) - monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec - ) + monkeypatch.setattr(prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec) logging_obj = MagicMock() logging_obj.async_get_chat_completion_prompt = AsyncMock( @@ -810,9 +789,7 @@ async def test_process_prompt_template_async_get_prompt_error_raises(proxy_loggi "get_prompt_callback_for_prompt", lambda *a, **kw: custom_logger, ) - monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec - ) + monkeypatch.setattr(prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec) logging_obj = MagicMock() logging_obj.async_get_chat_completion_prompt = AsyncMock(side_effect=RuntimeError("bad prompt")) with pytest.raises(RuntimeError): @@ -913,9 +890,7 @@ async def test_process_prompt_template_aresponses_swaps_model_and_merges_input(p "get_prompt_callback_for_prompt", lambda *a, **kw: custom_logger, ) - monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec - ) + monkeypatch.setattr(prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec) logging_obj = MagicMock() logging_obj.async_get_chat_completion_prompt = AsyncMock( @@ -1124,9 +1099,7 @@ async def test_pre_call_hook_still_runs_guardrail_managed_only_by_post_call_pipe }, } - await proxy_logging.pre_call_hook( - user_api_key_dict=make_user_api_key_auth(), data=data, call_type="completion" - ) + await proxy_logging.pre_call_hook(user_api_key_dict=make_user_api_key_auth(), data=data, call_type="completion") assert seen["count"] == 1 @@ -1289,11 +1262,7 @@ async def test_post_call_pipeline_block_keeps_guardrail_metadata_writes( monkeypatch.setattr( litellm, "callbacks", - [ - BlockingWriterGuardrail( - guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False - ) - ], + [BlockingWriterGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)], ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _post_call_pipeline_data() @@ -1379,9 +1348,7 @@ async def test_pre_call_pipeline_managed_parallel_guardrail_runs_exactly_once( }, } - await proxy_logging.pre_call_hook( - user_api_key_dict=make_user_api_key_auth(), data=data, call_type="completion" - ) + await proxy_logging.pre_call_hook(user_api_key_dict=make_user_api_key_auth(), data=data, call_type="completion") assert seen["count"] == 1 @@ -1433,14 +1400,20 @@ def _output_blocking_callbacks(seen: dict[str, object]) -> list[CustomGuardrail] seen["response"] = response raise HTTPException(status_code=400, detail={"error": "output blocked"}) - return [OutputBlockingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)] + return [ + OutputBlockingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False) + ] @pytest.mark.asyncio @pytest.mark.parametrize("pending_status", ["queued", "in_progress"]) async def test_post_call_success_hook_waits_for_pending_background_response_before_running_pipeline( - proxy_logging, make_user_api_key_auth, monkeypatch, caplog, pending_status -): + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + pending_status: str, +) -> None: seen: dict[str, object] = {} monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks(seen)) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) @@ -1464,8 +1437,11 @@ async def test_post_call_success_hook_waits_for_pending_background_response_befo @pytest.mark.asyncio @pytest.mark.parametrize("final_status", ["completed", "incomplete"]) async def test_post_call_success_hook_runs_pipeline_on_retrieved_background_response( - proxy_logging, make_user_api_key_auth, monkeypatch, final_status -): + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + final_status: str, +) -> None: seen: dict[str, object] = {} monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks(seen)) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) @@ -1486,7 +1462,9 @@ def _output_passing_callbacks() -> list[CustomGuardrail]: async def async_post_call_success_hook(self, data, user_api_key_dict, response): return response - return [OutputPassingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)] + return [ + OutputPassingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False) + ] def _claimed_post_call_pipeline_data( @@ -1519,7 +1497,7 @@ def _claimed_post_call_pipeline_data( @pytest.fixture -def clear_policy_registry(): +def clear_policy_registry() -> Iterator[None]: from litellm.proxy.policy_engine.policy_registry import get_policy_registry yield @@ -1528,8 +1506,11 @@ def clear_policy_registry(): @pytest.mark.asyncio async def test_pending_background_response_withdraws_the_deferred_policy_claims( - proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry -): + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + clear_policy_registry: None, +) -> None: monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _claimed_post_call_pipeline_data("response-governance") @@ -1546,8 +1527,12 @@ async def test_pending_background_response_withdraws_the_deferred_policy_claims( @pytest.mark.asyncio async def test_pending_background_response_warns_when_the_deferred_policy_was_matched_through_a_tag( - proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry, caplog -): + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + clear_policy_registry: None, + caplog: pytest.LogCaptureFixture, +) -> None: monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _claimed_post_call_pipeline_data("response-governance", policy_source="tag:governed+model:m") @@ -1559,19 +1544,21 @@ async def test_pending_background_response_warns_when_the_deferred_policy_was_ma assert out.status == "queued" assert "policy_sources" not in data["metadata"] - assert [ - message for message in _warnings(caplog) if "response-governance (tag:governed+model:m)" in message - ] == [ + assert [message for message in _warnings(caplog) if "through a request tag" in message] == [ "Policy engine: background response resp_bg matched post_call policies through a request tag at submit; " "retrieval re-matches only the key, team, and model scopes, so a tag carried in the request body " - "does not govern the completed response: response-governance (tag:governed+model:m)" + "does not govern the completed response: response-governance" ] @pytest.mark.asyncio async def test_pending_background_response_matched_through_its_model_does_not_warn( - proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry, caplog -): + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + clear_policy_registry: None, + caplog: pytest.LogCaptureFixture, +) -> None: monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) @@ -1587,12 +1574,17 @@ async def test_pending_background_response_matched_through_its_model_does_not_wa @pytest.mark.asyncio async def test_pending_background_response_keeps_the_claim_of_a_policy_that_runs_outside_its_pipeline( - proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry -): + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + clear_policy_registry: None, +) -> None: monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _claimed_post_call_pipeline_data( - "input-and-output-governance", "response-governance", extra_guardrails={"input-and-output-governance": ["gr-pre"]} + "input-and-output-governance", + "response-governance", + extra_guardrails={"input-and-output-governance": ["gr-pre"]}, ) await proxy_logging.post_call_success_hook( @@ -1606,8 +1598,11 @@ async def test_pending_background_response_keeps_the_claim_of_a_policy_that_runs @pytest.mark.asyncio async def test_pending_background_response_keeps_the_claim_of_a_default_on_guardrail_that_ran_pre_call( - proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry -): + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + clear_policy_registry: None, +) -> None: class DualStageGuardrail(CustomGuardrail): async def async_post_call_success_hook(self, data, user_api_key_dict, response): return response @@ -1630,8 +1625,11 @@ async def test_pending_background_response_keeps_the_claim_of_a_default_on_guard @pytest.mark.asyncio async def test_retrieved_background_response_keeps_the_policy_claim_once_its_pipeline_ran( - proxy_logging, make_user_api_key_auth, monkeypatch, clear_policy_registry -): + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + clear_policy_registry: None, +) -> None: monkeypatch.setattr(litellm, "callbacks", _output_passing_callbacks()) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _claimed_post_call_pipeline_data("response-governance") @@ -1647,8 +1645,11 @@ async def test_retrieved_background_response_keeps_the_policy_claim_once_its_pip @pytest.mark.asyncio async def test_pre_call_hook_stays_quiet_on_background_request_with_post_call_pipeline( - proxy_logging, make_user_api_key_auth, monkeypatch, caplog -): + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: monkeypatch.setattr(litellm, "callbacks", []) data = _post_call_pipeline_data(background=True) @@ -1709,7 +1710,9 @@ def test_streamable_post_call_pipelines_keeps_supported_and_drops_unsupported( steps=[PipelineStep(guardrail="gr-post", on_fail="next"), PipelineStep(guardrail="gr-native", on_fail="block")], ) pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="gr-native", on_fail="block")]) - data = {"metadata": {"_guardrail_pipelines": [("governed", governed), ("ungoverned", ungoverned), ("req", pre_call)]}} + data = { + "metadata": {"_guardrail_pipelines": [("governed", governed), ("ungoverned", ungoverned), ("req", pre_call)]} + } with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): streamable = _streamable_post_call_pipelines(data, make_user_api_key_auth(request_route="/v1/chat/completions")) @@ -2009,7 +2012,9 @@ def _rewriting_stream_guardrail(transform: Callable[[Dict[str, Any]], Dict[str, async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): return {**inputs, **transform(inputs)} - return RewritingStreamGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False) + return RewritingStreamGuardrail( + guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False + ) def _tool_call_stream_chunks() -> List[Any]: @@ -2188,11 +2193,38 @@ async def test_streaming_iterator_hook_pipeline_releases_originals_on_unresolvab def _anthropic_sse_chunks() -> List[bytes]: events = [ - ("message_start", {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "m", "content": [], "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 0}}}), - ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), - ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello world"}}), + ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "m", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + ), + ( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello world"}}, + ), ("content_block_stop", {"type": "content_block_stop", "index": 0}), - ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 2}}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 2}, + }, + ), ("message_stop", {"type": "message_stop"}), ] return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events] @@ -2259,7 +2291,13 @@ async def test_streaming_iterator_hook_pipeline_delivers_text_rewrite_on_anthrop assert "hello [MASKED]" in raw assert "hello world" not in raw assert raw.count("event: content_block_delta") == 1 - for expected_event in ("message_start", "content_block_start", "content_block_stop", "message_delta", "message_stop"): + for expected_event in ( + "message_start", + "content_block_start", + "content_block_stop", + "message_delta", + "message_stop", + ): assert f"event: {expected_event}" in raw @@ -2332,9 +2370,7 @@ async def test_per_chunk_streaming_hook_skips_pipeline_managed_guardrail( managed = UnifiedRecordingGuardrail( guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True ) - free = RecordingGuardrail( - guardrail_name="gr-free", event_hook=GuardrailEventHooks.post_call, default_on=True - ) + free = RecordingGuardrail(guardrail_name="gr-free", event_hook=GuardrailEventHooks.post_call, default_on=True) monkeypatch.setattr(litellm, "callbacks", [managed, free]) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) data = _post_call_pipeline_data(stream=True) From c66ae07e3bb97e875d88096458e5de80b351828c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:11:01 -0700 Subject: [PATCH 164/411] fix(hosted_vllm): reject image edit params vLLM-Omni ignores --- .../hosted_vllm/image_edit/transformation.py | 9 ++++ litellm/utils.py | 2 +- ...t_hosted_vllm_image_edit_transformation.py | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/litellm/llms/hosted_vllm/image_edit/transformation.py b/litellm/llms/hosted_vllm/image_edit/transformation.py index fa8c1dbfcfd..3b8cc437168 100644 --- a/litellm/llms/hosted_vllm/image_edit/transformation.py +++ b/litellm/llms/hosted_vllm/image_edit/transformation.py @@ -3,8 +3,17 @@ from typing import Final from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig from litellm.secret_managers.main import get_secret_str +PARAMS_VLLM_OMNI_DOES_NOT_ACCEPT: Final = frozenset({"mask", "quality", "input_fidelity"}) + class HostedVLLMImageEditConfig(OpenAIImageEditConfig): + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseImageEditConfig contract + return [ # mutable-ok: BaseImageEditConfig returns list + param + for param in super().get_supported_openai_params(model) + if param not in PARAMS_VLLM_OMNI_DOES_NOT_ACCEPT + ] + def validate_environment( self, headers: dict, # mutable-ok: BaseImageEditConfig contract diff --git a/litellm/utils.py b/litellm/utils.py index 0f2b2f9042d..fc530f14e28 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9239,7 +9239,7 @@ class ProviderConfigManager: from litellm.llms.openai.image_edit import get_openai_image_edit_config return get_openai_image_edit_config(model=model) - if LlmProviders.HOSTED_VLLM == provider: + elif LlmProviders.HOSTED_VLLM == provider: from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config return get_hosted_vllm_image_edit_config(model=model) diff --git a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py index 5646f3d9d13..bc0ea23e249 100644 --- a/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/image_edit/test_hosted_vllm_image_edit_transformation.py @@ -110,3 +110,46 @@ def test_image_edit_posts_multipart_to_vllm_omni(): assert f'name="model"\r\n\r\n{MODEL}'.encode() in request.content assert b'name="prompt"\r\n\r\nadd a hat' in request.content assert b'name="seed"\r\n\r\n42' in request.content + + +@pytest.mark.parametrize("param", ["mask", "quality", "input_fidelity"]) +def test_params_vllm_omni_ignores_are_not_advertised(param: str): + supported = HostedVLLMImageEditConfig().get_supported_openai_params(MODEL) + + assert param not in supported + assert {"image", "prompt", "n", "size", "response_format", "background", "user"} <= set(supported) + + +def test_image_edit_rejects_quality_unless_dropped(): + captured: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response(200, json={"created": 1712697600, "data": [{"b64_json": "aW1n"}]}) + + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))) + + with pytest.raises(litellm.UnsupportedParamsError, match="quality"): + litellm.image_edit( + model=f"hosted_vllm/{MODEL}", + image=PNG_BYTES, + prompt="add a hat", + api_base="http://localhost:8091", + client=client, + quality="low", + ) + assert captured == [] + + litellm.image_edit( + model=f"hosted_vllm/{MODEL}", + image=PNG_BYTES, + prompt="add a hat", + api_base="http://localhost:8091", + client=client, + quality="low", + drop_params=True, + ) + + assert len(captured) == 1 + assert b'name="quality"' not in captured[0].content + assert b'name="prompt"\r\n\r\nadd a hat' in captured[0].content From 8e3052ff650460d8ba226381446c6a9b95e87cd8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:20:57 -0700 Subject: [PATCH 165/411] fix(azure_ai): honour global drop_params for image params MAI cannot serve get_optional_params_image_gen only forwarded the per-call drop_params flag to provider configs, so litellm_settings drop_params: true never dropped the n the MAI generations endpoint ignores. The MAI edits config also advertised and forwarded size, which that endpoint ignores. Non-numeric and non-positive n now surface as a 400 instead of a 500 or a pass-through. --- .../azure_ai/image_edit/mai_transformation.py | 62 +---------------- .../image_generation/mai_transformation.py | 10 ++- litellm/utils.py | 2 +- .../test_mai_image_edit_transformation.py | 67 ++++++++++--------- .../test_mai_image_generation.py | 38 ++++++++++- 5 files changed, 83 insertions(+), 96 deletions(-) diff --git a/litellm/llms/azure_ai/image_edit/mai_transformation.py b/litellm/llms/azure_ai/image_edit/mai_transformation.py index e639c20292b..55b179e9591 100644 --- a/litellm/llms/azure_ai/image_edit/mai_transformation.py +++ b/litellm/llms/azure_ai/image_edit/mai_transformation.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final import httpx from httpx._types import RequestFiles @@ -13,7 +13,6 @@ from litellm.llms.azure_ai.image_generation.mai_transformation import ( from litellm.llms.openai.common_utils import OpenAIError from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.llms.openai import FileTypes from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ImageResponse @@ -26,65 +25,8 @@ if TYPE_CHECKING: class AzureFoundryMAIImageEditConfig(OpenAIImageEditConfig): """Azure AI Foundry MAI image editing (e.g. MAI-Image-2.5).""" - DEFAULT_SIZE = "1024x1024" - def get_supported_openai_params(self, model: str) -> list: - return ["prompt", "image", "model", "n", "size"] - - def map_openai_params( - self, - image_edit_optional_params: ImageEditOptionalRequestParams, - model: str, - drop_params: bool, - ) -> dict: - optional_params: Final[dict[str, Any]] = {} - supported_params: Final = self.get_supported_openai_params(model) - - for key, value in dict(image_edit_optional_params).items(): - if value is None or key in optional_params: - continue - - if key in supported_params: - if key == "size" and value: - size_param = cast(str, value) - self._validate_size_param(size_param) - optional_params[key] = size_param - else: - optional_params[key] = value - elif not drop_params: - raise ValueError( - f"Parameter {key} is not supported for model {model}. " - f"Supported parameters are {supported_params}. " - f"Set drop_params=True to drop unsupported parameters." - ) - - if "size" not in optional_params: - optional_params["size"] = self.DEFAULT_SIZE - - return optional_params - - def _validate_size_param(self, size: str) -> None: - known_sizes: Final = { - "1024x1024", - "1792x1024", - "1024x1792", - "512x512", - "256x256", - } - - if size in known_sizes: - return - - if "x" in size: - try: - tuple(map(int, size.lower().split("x", 1))) - return - except ValueError: - raise ValueError(f"Invalid size format: '{size}'. Expected format 'WIDTHxHEIGHT' (e.g., '1024x1024').") - - raise ValueError( - f"Unsupported size value: '{size}'. Use a known size (e.g., '1024x1024') or a custom 'WIDTHxHEIGHT' string." - ) + return ["prompt", "image", "model", "n"] def validate_environment( self, diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 7f7a1044689..67b1a8bcab3 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -151,7 +151,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): if k in supported_params: if k == "size" and v: self._map_size_param(v, optional_params, model) - elif k == "n" and v is not None and int(v) > self.MAX_IMAGES_PER_REQUEST: + elif k == "n" and v is not None and self._image_count(v, model) != self.MAX_IMAGES_PER_REQUEST: if not drop_params: raise self._unsupported( model, @@ -185,6 +185,14 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): def _unsupported(model: str, message: str) -> UnsupportedParamsError: return UnsupportedParamsError(message=message, llm_provider="azure_ai", model=model) + def _image_count(self, n: object, model: str) -> int: + if isinstance(n, int): + return n + try: + return int(str(n)) + except ValueError: + raise self._unsupported(model, f"n={n!r} is not a whole number of images for model {model}.") + def _map_size_param(self, size: str, optional_params: dict, model: str) -> None: size_mapping: Final = { "1024x1024": (1024, 1024), diff --git a/litellm/utils.py b/litellm/utils.py index bc2f4a86f12..63ad4bdd945 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3396,7 +3396,7 @@ def get_optional_params_image_gen( non_default_params=non_default_params, optional_params=optional_params, model=model or "", - drop_params=drop_params if drop_params is not None else False, + drop_params=litellm.drop_params is True or drop_params is True, ) elif ( custom_llm_provider == "openai" diff --git a/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py b/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py index 284a912d9a4..75e046825a3 100644 --- a/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py +++ b/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py @@ -6,6 +6,7 @@ import pytest import litellm +from litellm.images.utils import ImageEditRequestUtils from litellm.llms.azure_ai.image_edit import ( AzureFoundryMAIImageEditConfig, get_azure_ai_image_edit_config, @@ -70,44 +71,48 @@ class TestAzureMAIImageEdit: assert "/mai/v1/images/edits" in url assert "api-version=preview" in url - def test_map_openai_params_keeps_size(self): - config = AzureFoundryMAIImageEditConfig() - optional_params = config.map_openai_params( - image_edit_optional_params={"size": "1792x1024", "n": 1}, + def test_get_optional_params_image_edit_size_raises_400(self, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError, match="size") as exc_info: + ImageEditRequestUtils.get_optional_params_image_edit( + model="MAI-Image-2.5", + image_edit_provider_config=AzureFoundryMAIImageEditConfig(), + image_edit_optional_params={"size": "1024x1024", "n": 1}, + ) + assert exc_info.value.status_code == 400 + + def test_get_optional_params_image_edit_size_dropped_with_drop_params(self, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + optional_params = ImageEditRequestUtils.get_optional_params_image_edit( model="MAI-Image-2.5", + image_edit_provider_config=AzureFoundryMAIImageEditConfig(), + image_edit_optional_params={"size": "1024x1024", "n": 1}, drop_params=True, ) - assert optional_params["size"] == "1792x1024" + assert "size" not in optional_params assert optional_params["n"] == 1 - assert "width" not in optional_params - assert "height" not in optional_params - def test_map_openai_params_defaults_size(self): - config = AzureFoundryMAIImageEditConfig() - optional_params = config.map_openai_params( - image_edit_optional_params={}, + def test_get_optional_params_image_edit_without_size_forwards_nothing_extra(self, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + optional_params = ImageEditRequestUtils.get_optional_params_image_edit( model="MAI-Image-2.5", - drop_params=True, + image_edit_provider_config=AzureFoundryMAIImageEditConfig(), + image_edit_optional_params={}, ) - assert optional_params["size"] == "1024x1024" + assert optional_params == {} - def test_map_openai_params_unsupported_size_raises(self): - config = AzureFoundryMAIImageEditConfig() - with pytest.raises(ValueError, match="Unsupported size value: 'auto'"): - config.map_openai_params( - image_edit_optional_params={"size": "auto"}, - model="MAI-Image-2.5", - drop_params=True, - ) - - def test_map_openai_params_invalid_size_format_raises(self): - config = AzureFoundryMAIImageEditConfig() - with pytest.raises(ValueError, match="Invalid size format: '1024xabc'"): - config.map_openai_params( - image_edit_optional_params={"size": "1024xabc"}, - model="MAI-Image-2.5", - drop_params=True, + def test_image_edit_size_surfaces_as_400(self, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.image_edit( + model="azure_ai/MAI-Image-2.5", + image=io.BytesIO(b"fake-image-bytes"), + prompt="Turn this into a studio product shot", + size="1024x1024", + api_key="test-key", + api_base="https://my-resource.services.ai.azure.com", ) + assert exc_info.value.status_code == 400 def test_transform_image_edit_request_uses_image_field(self): config = AzureFoundryMAIImageEditConfig() @@ -117,14 +122,14 @@ class TestAzureMAIImageEdit: model="MAI-Image-2.5", prompt="Turn this into a studio product shot", image=image_bytes, - image_edit_optional_request_params={"size": "1024x1024", "n": 1}, + image_edit_optional_request_params={"n": 1}, litellm_params={}, headers={}, ) assert data["model"] == "MAI-Image-2.5" assert data["prompt"] == "Turn this into a studio product shot" - assert data["size"] == "1024x1024" + assert "size" not in data assert data["n"] == 1 assert len(files) == 1 assert files[0][0] == "image" diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py index a0bc3fb00b7..b8632a7ca5c 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py @@ -253,8 +253,8 @@ class TestAzureMAIImageGeneration: assert optional_params["width"] == 1792 assert optional_params["height"] == 1024 - @pytest.mark.parametrize("n", [2, 4, "2"]) - def test_map_openai_params_multi_image_n_raises(self, n): + @pytest.mark.parametrize("n", [2, 4, "2", 0, -1]) + def test_map_openai_params_n_other_than_one_raises(self, n): config = AzureFoundryMAIImageGenerationConfig() with pytest.raises(UnsupportedParamsError, match="returns exactly 1 image per request"): config.map_openai_params( @@ -264,6 +264,38 @@ class TestAzureMAIImageGeneration: drop_params=False, ) + def test_map_openai_params_non_numeric_n_raises_400(self): + config = AzureFoundryMAIImageGenerationConfig() + with pytest.raises(UnsupportedParamsError, match="not a whole number of images") as exc_info: + config.map_openai_params( + non_default_params={"n": "abc"}, + optional_params={}, + model="MAI-Image-2.5", + drop_params=False, + ) + assert exc_info.value.status_code == 400 + + def test_get_optional_params_image_gen_global_drop_params_drops_multi_image_n(self, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", True) + optional_params = get_optional_params_image_gen( + model="MAI-Image-2.5", + n=4, + custom_llm_provider="azure_ai", + provider_config=AzureFoundryMAIImageGenerationConfig(), + ) + assert "n" not in optional_params + assert optional_params["width"] == 1024 + + def test_get_optional_params_image_gen_without_any_drop_params_still_raises(self, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(UnsupportedParamsError, match="returns exactly 1 image per request"): + get_optional_params_image_gen( + model="MAI-Image-2.5", + n=4, + custom_llm_provider="azure_ai", + provider_config=AzureFoundryMAIImageGenerationConfig(), + ) + def test_map_openai_params_multi_image_n_dropped_with_drop_params(self): config = AzureFoundryMAIImageGenerationConfig() optional_params = config.map_openai_params( @@ -284,7 +316,7 @@ class TestAzureMAIImageGeneration: ) assert optional_params["n"] == 1 - @pytest.mark.parametrize("params", [{"n": 2}, {"size": "512x512"}, {"size": "1792x1024"}]) + @pytest.mark.parametrize("params", [{"n": 2}, {"n": "abc"}, {"size": "512x512"}, {"size": "1792x1024"}]) def test_image_generation_rejected_params_surface_as_400(self, params): with pytest.raises(litellm.BadRequestError) as exc_info: litellm.image_generation( From b5d40bef2b830b6d5ccbb3f3fae74eb10968b9cd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:25:04 -0700 Subject: [PATCH 166/411] fix(token_counter): bound offloaded token counts with a dedicated capacity limiter Replace the thread semaphore around HuggingFace encodes with an anyio CapacityLimiter applied at every offloaded count site through offload_token_count, so waiting counts no longer hold slots in the shared 40-thread pool and inline counts on the event loop never block on the bound. Rename TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES to TOKEN_COUNTER_MAX_CONCURRENT_COUNTS --- litellm/constants.py | 2 +- litellm/litellm_core_utils/token_counter.py | 21 ++++++++++----- .../hooks/parallel_request_limiter_v3.py | 4 +-- litellm/proxy/proxy_server.py | 6 ++--- litellm/proxy/utils.py | 4 +-- litellm/router.py | 5 ++-- .../complexity_router/complexity_router.py | 6 ++--- .../io_token_rate_limit_check.py | 4 +-- .../prompt_caching_deployment_check.py | 6 ++--- .../litellm_core_utils/test_token_counter.py | 26 +++++++++++++------ 10 files changed, 52 insertions(+), 32 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 05d442dd070..0bd7def539d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -397,7 +397,7 @@ TOKEN_COUNTER_MAX_EXACT_CHARS: Final = get_env_int_in_range( minimum=1, maximum=1_000_000_000, ) -TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES: Final = 4 +TOKEN_COUNTER_MAX_CONCURRENT_COUNTS: Final = 4 MAX_TILE_WIDTH: Final = int(os.getenv("MAX_TILE_WIDTH", 512)) MAX_TILE_HEIGHT: Final = int(os.getenv("MAX_TILE_HEIGHT", 512)) OPENAI_FILE_SEARCH_COST_PER_1K_CALLS: Final = float(os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000)) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8624e79faea..e71ce5dea9c 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,13 +3,14 @@ import base64 import io import struct -import threading -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from typing import Final, Literal, cast +import anyio import httpx import tiktoken from tokenizers import Tokenizer +from typing_extensions import ParamSpec, TypeVar import litellm from litellm import verbose_logger @@ -23,9 +24,10 @@ from litellm.constants import ( MAX_TILE_HEIGHT, MAX_TILE_WIDTH, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, - TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES, + TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, TOKEN_COUNTER_MAX_EXACT_CHARS, ) +from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.default_encoding import encoding as default_encoding from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client @@ -322,7 +324,15 @@ Type for a function that counts tokens in a string. """ EXTRAPOLATION_SAMPLES: Final = 16 -_HF_ENCODE_SLOTS: Final = threading.BoundedSemaphore(TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES) +T_ParamSpec: Final = ParamSpec("T_ParamSpec") +T_Retval = TypeVar("T_Retval") +_COUNT_OFFLOAD_LIMITER: Final = anyio.CapacityLimiter(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS) + + +def offload_token_count( + function: Callable[T_ParamSpec, T_Retval], +) -> Callable[T_ParamSpec, Awaitable[T_Retval]]: + return asyncify(function, limiter=_COUNT_OFFLOAD_LIMITER) def _get_tiktoken_count_function( @@ -590,8 +600,7 @@ def _get_exact_count_function( tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: - with _HF_ENCODE_SLOTS: - return len(tokenizer.encode_batch_fast([text])[0]) + return len(tokenizer.encode_batch_fast([text])[0]) return count_tokens elif tokenizer_json["type"] == "openai_tokenizer": diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 74266801e50..117be770953 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -28,10 +28,10 @@ from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) +from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( ESTIMATED_OUTPUT_TOKENS_FIELD, @@ -3304,7 +3304,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit=min_configured_otpm_limit, call_type=call_type, ) - raw_estimated_input_tokens: Final = await asyncify(self._estimate_precise_input_tokens)( + raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)( data=data, model=requested_model, call_type=call_type ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 77354e9885f..0e3e5c251c8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -67,6 +67,7 @@ from litellm.litellm_core_utils.litellm_logging import ( ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.proxy._types import ( UI_TEAM_ID, CallbackDelete, @@ -274,7 +275,6 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) -from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, @@ -12713,7 +12713,7 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False) CustomHuggingfaceTokenizer | None, model_info.get("custom_tokenizer", None), ) - _tokenizer_used: Final = await asyncify(litellm.utils._select_tokenizer)( + _tokenizer_used: Final = await offload_token_count(litellm.utils._select_tokenizer)( model=model_to_use, custom_tokenizer=custom_tokenizer ) @@ -12728,7 +12728,7 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False) counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None ) - total_tokens: Final = await asyncify(litellm.token_counter)( + total_tokens: Final = await offload_token_count(litellm.token_counter)( model=model_to_use, text=prompt, messages=counted_messages, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1014c2b9b6a..59c09081890 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -91,7 +91,6 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert -from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import ( coerce_token_limit, independent_snapshot, @@ -100,6 +99,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.llms import load_guardrail_translation_mappings from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( @@ -2570,7 +2570,7 @@ class ProxyLogging: original_exception=original_exception, ) - request_data.update(await asyncify(_failure_fields_to_lift)(request_data)) + request_data.update(await offload_token_count(_failure_fields_to_lift)(request_data)) # Remove before callbacks iterate — not serialisable request_data.pop("litellm_logging_obj", None) diff --git a/litellm/router.py b/litellm/router.py index 95cabfad4bd..73abd97d80d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -55,7 +55,7 @@ from litellm.constants import ( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.asyncify import asyncify, run_async_function +from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, coerce_token_limit, @@ -86,6 +86,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( mask_credentials_in_payload, mask_sensitive_structure, ) +from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, vector_store_request_metadata, @@ -11958,7 +11959,7 @@ class Router: try: if not self._pre_call_checks_need_token_count(model, healthy_deployments): return None - return await asyncify(self._count_pre_call_check_tokens)( + return await offload_token_count(self._count_pre_call_check_tokens)( messages=cast(list[dict[str, str]] | None, messages), # cast-ok: forwarded to the sync counter input=cast(str | list | None, input), # cast-ok: forwarded to the sync counter request_kwargs=request_kwargs, diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 7d4497fb6f7..cc1bb54255b 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2486,14 +2486,14 @@ class ComplexityRouter(CustomLogger): """Real-tokenizer count of the resolved messages plus the out-of-band carriers, off the event loop; None when counting fails, and the gate then leaves the placement alone.""" import litellm - from litellm.litellm_core_utils.asyncify import asyncify + from litellm.litellm_core_utils.token_counter import offload_token_count out_of_band: Final = self._out_of_band_request_text(request_kwargs) try: - counted: Final = await asyncify(litellm.token_counter)( + counted: Final = await offload_token_count(litellm.token_counter)( messages=cast(list, resolved_messages) # cast-ok: token_counter only iterates the sequence ) - return counted + (await asyncify(litellm.token_counter)(text=out_of_band) if out_of_band else 0) + return counted + (await offload_token_count(litellm.token_counter)(text=out_of_band) if out_of_band else 0) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.debug("ComplexityRouter: context-window token count failed. Got - %s", e) return None diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index dca5a2be845..fbd3e18e357 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -21,7 +21,7 @@ import litellm from litellm import token_counter from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.types.router import RouterCacheEnum, RouterErrors from litellm.utils import get_utc_datetime @@ -467,7 +467,7 @@ async def async_io_token_pre_call_check( request_kwargs: Final = get_io_token_rate_limit_request_kwargs() _model: Final = (deployment.get("litellm_params") or {}).get("model") or "" - estimated_input: Final = await asyncify(_estimate_input_tokens)(request_kwargs, model=_model) + estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model) max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) dt: Final = get_utc_datetime() diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index 9ea528b9700..0589e290b47 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -14,7 +14,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, ) from litellm.integrations.custom_logger import CustomLogger, Span -from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes, StandardLoggingPayload from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt @@ -62,7 +62,7 @@ class PromptCachingDeploymentCheck(CustomLogger): if request_kwargs is not None and request_kwargs.get("_target_order") is not None: return healthy_deployments - if messages is not None and await asyncify(is_prompt_caching_valid_prompt)( + if messages is not None and await offload_token_count(is_prompt_caching_valid_prompt)( messages=messages, model=model, min_token_count=_get_min_token_count_for_deployments(healthy_deployments), @@ -140,7 +140,7 @@ class PromptCachingDeploymentCheck(CustomLogger): return ## PROMPT CACHING - cache model id, if prompt caching valid prompt + provider - if await asyncify(is_prompt_caching_valid_prompt)( + if await offload_token_count(is_prompt_caching_valid_prompt)( model=model, messages=cast(list[AllMessageValues], messages), ): diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 655be5146cd..7cc7c6c763c 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -5,10 +5,10 @@ import importlib import threading import time import traceback -from concurrent.futures import ThreadPoolExecutor from typing import Final from unittest.mock import MagicMock +import anyio.to_thread import pytest import tiktoken @@ -18,12 +18,13 @@ import litellm from litellm import create_pretrained_tokenizer, decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.constants import TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES +from litellm.constants import TOKEN_COUNTER_MAX_CONCURRENT_COUNTS from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( _get_exact_count_function, _get_extrapolating_count_function, _get_tiktoken_count_function, + offload_token_count, ) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new from tests.large_text import text @@ -181,16 +182,25 @@ class _SlowEncoder: return [[0] * len(text) for text in texts] -def test_huggingface_counts_run_at_most_the_configured_number_at_once(): +@pytest.mark.asyncio +async def test_offloaded_counts_do_not_borrow_from_the_shared_thread_pool(): encoder: Final = _SlowEncoder() count: Final = _get_exact_count_function(None, {"type": "huggingface_tokenizer", "tokenizer": encoder}) - burst: Final = 2 * TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES + shared_pool: Final = anyio.to_thread.current_default_thread_limiter() + burst: Final = 2 * TOKEN_COUNTER_MAX_CONCURRENT_COUNTS - with ThreadPoolExecutor(max_workers=burst) as pool: - counts: Final = tuple(pool.map(count, ["abc"] * burst)) + async def shared_pool_borrowed_until_done(counting: asyncio.Future[list[int]]) -> tuple[int, ...]: + if counting.done(): + return () + await asyncio.sleep(0.01) + return (shared_pool.borrowed_tokens, *await shared_pool_borrowed_until_done(counting)) - assert counts == (3,) * burst - assert 1 < encoder.peak_in_flight <= TOKEN_COUNTER_MAX_CONCURRENT_HF_ENCODES + counting: Final = asyncio.ensure_future(asyncio.gather(*(offload_token_count(count)("abc") for _ in range(burst)))) + borrowed: Final = await shared_pool_borrowed_until_done(counting) + + assert await counting == [3] * burst + assert len(borrowed) > 1 and max(borrowed) == 0 + assert 1 < encoder.peak_in_flight <= TOKEN_COUNTER_MAX_CONCURRENT_COUNTS def test_token_counter_applies_the_default_cap(): From 2bca7ff673f07812c544ce42a165dda931b8a730 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:27:14 -0700 Subject: [PATCH 167/411] fix(anthropic): replay OpenAI encrypted reasoning byte for byte behind /v1/messages Reasoning items the Responses API returned for a /v1/messages turn were rebuilt from their summary text on every replay, so the prompt the model saw changed between turns and the prompt cache never matched. The bridge now asks for reasoning.encrypted_content, carries it in the thinking signature (or as a redacted_thinking block when there is no summary), and replays it verbatim as the reasoning item's encrypted_content. Anthropic replay paths drop those tagged blocks so a cross-model resume never forwards OpenAI bytes to Anthropic --- .../transformation.py | 6 +- .../prompt_templates/common_utils.py | 80 +++++++-- .../prompt_templates/factory.py | 10 +- litellm/llms/anthropic/common_utils.py | 13 +- .../responses_adapters/streaming_iterator.py | 66 ++++++- .../responses_adapters/transformation.py | 84 +++++---- ...ore_utils_prompt_templates_common_utils.py | 64 +++++++ ...llm_core_utils_prompt_templates_factory.py | 12 +- ...t_responses_adapters_streaming_iterator.py | 74 +++++++- .../test_responses_adapters_transformation.py | 168 +++++++++++++++++- .../anthropic/test_anthropic_common_utils.py | 24 +++ 11 files changed, 535 insertions(+), 66 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 4fe069b0b7d..135f34afe46 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -25,7 +25,7 @@ import litellm from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( - responses_reasoning_item_from_thinking_blocks, + responses_reasoning_items_from_thinking_blocks, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( @@ -129,8 +129,8 @@ def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: return stored raw_blocks: Final = msg.get("thinking_blocks") or () blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json - from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks) - return [] if from_thinking is None else [dict(from_thinking)] # mutable-ok: API message payload + replayed: Final = responses_reasoning_items_from_thinking_blocks(blocks) + return [dict(item) for item in replayed] # mutable-ok: API message payload def _build_reasoning_item( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 00b80839dde..918cb982773 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1823,14 +1823,11 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]: return None, message_content -def _readable_thinking_text( - block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, -) -> str: +def _readable_thinking_text(block: Mapping[str, object]) -> str: """The text a chat model can read back, empty for redacted blocks and malformed ones.""" if block.get("type") != "thinking": return "" - thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking") # cast-ok: narrowed by the type tag - return str(thinking or "") + return str(block.get("thinking") or "") def reasoning_content_from_thinking_blocks( @@ -1843,24 +1840,83 @@ def reasoning_content_from_thinking_blocks( return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block))) -def responses_reasoning_item_from_thinking_blocks( - thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], -) -> ChatCompletionReasoningItem | None: - """Build a Responses API `reasoning` input item from Anthropic thinking blocks. +ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:" - The item carries no `id`: the Responses API rejects an empty one and 404s on any id it - did not mint itself, while an item without an id is always accepted. + +def encrypted_reasoning_signature(encrypted_content: str) -> str: + """The opaque value a Responses API reasoning item's `encrypted_content` travels in. + + Anthropic clients echo a thinking block's `signature` and a redacted block's `data` + back verbatim, so either field can carry the encrypted reasoning across turns; the + prefix tells the two apart from a signature Anthropic minted. """ + return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}" + + +def encrypted_content_from_signature(signature: object) -> str | None: + if not isinstance(signature, str) or not signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX): + return None + return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None + + +def _encrypted_content_of_block(block: Mapping[str, object]) -> str | None: + match block.get("type"): + case "thinking": + return encrypted_content_from_signature(block.get("signature")) + case "redacted_thinking": + return encrypted_content_from_signature(block.get("data")) + case _: + return None + + +def is_encrypted_reasoning_block(block: object) -> bool: + """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. + + Only the Responses API that minted the content can read it back, so an Anthropic + backend has to drop such a block rather than fail signature verification on it. + """ + if not isinstance(block, Mapping): + return False + mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + return _encrypted_content_of_block(mapping) is not None + + +def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: + index, block = indexed_block + return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary" + + +def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None: summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text) - for block in thinking_blocks + for block in group if (text := _readable_thinking_text(block)) ] + encrypted_content: Final = _encrypted_content_of_block(group[0]) + if encrypted_content is not None: + return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content) if not summary: return None return ChatCompletionReasoningItem(type="reasoning", summary=summary) +def responses_reasoning_items_from_thinking_blocks( + thinking_blocks: Iterable[Mapping[str, object]], +) -> tuple[ChatCompletionReasoningItem, ...]: + """Build Responses API `reasoning` input items from Anthropic thinking blocks. + + A block carrying encrypted reasoning replays the item it came from byte for byte; + a run of plain thinking blocks collapses into one summary-only item. No item carries + an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty + one, while an item without an id is always accepted. + """ + return tuple( + item + for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key) + if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None + ) + + def _parse_content_for_reasoning( message_text: str | None, ) -> tuple[str | None, str | None]: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 56c1d605700..ece619e3883 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -46,6 +46,7 @@ from litellm.types.utils import GenericImageParsingChunk from .common_utils import ( convert_content_list_to_str, infer_content_type_from_url_and_content, + is_encrypted_reasoning_block, is_non_content_values_set, parse_tool_call_arguments, ) @@ -2299,13 +2300,16 @@ def sanitize_messages_for_tool_calling( def _is_unsignable_thinking_block(block: object) -> bool: - """A `thinking` block that Anthropic cannot accept on input. + """A thinking block that Anthropic cannot accept on input. Anthropic verifies the thinking signature cryptographically, so a block whose signature is null, empty, or missing (e.g. from an open-source reasoning model) - is rejected with a 400 and must be dropped rather than blanked or repaired. - `redacted_thinking` blocks carry no signature and are always kept. + is rejected with a 400 and must be dropped rather than blanked or repaired, and + so is a block whose signature or data carries another provider's encrypted + reasoning. A `redacted_thinking` block Anthropic minted is always kept. """ + if is_encrypted_reasoning_block(block): + return True if not isinstance(block, dict) or block.get("type") != "thinking": return False signature: Final = block.get("signature") diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 2b57883cc13..f46227239d8 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -21,6 +21,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_file_ids_from_messages, + is_encrypted_reasoning_block, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, @@ -1235,8 +1236,10 @@ def strip_empty_content_blocks_from_anthropic_messages( on the unified ``/v1/messages`` path. ``/v1/chat/completions`` already handles this in ``anthropic_messages_pt``; this helper provides the equivalent guarantee for the native Anthropic Messages path. - ``redacted_thinking`` blocks are never touched: they carry opaque - ``data`` instead of thinking text. + A thinking or ``redacted_thinking`` block whose signature or data carries + another provider's encrypted reasoning (a turn served by the Responses API + bridge) is dropped too, since Anthropic cannot verify it; every other + ``redacted_thinking`` block is left alone. Messages whose content is a list and becomes empty after stripping are omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`. @@ -1249,7 +1252,11 @@ def strip_empty_content_blocks_from_anthropic_messages( out.append(m) continue content = m["content"] - filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_thinking_block(b)] + filtered = [ # mutable-ok: rebuilt message content list + b + for b in content + if not _is_empty_text_block(b) and not is_empty_thinking_block(b) and not is_encrypted_reasoning_block(b) + ] if len(filtered) == len(content): out.append(m) elif filtered: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 2e0a6a9df8f..f753e87fee3 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -9,13 +9,19 @@ from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( refusal_stop_details, responses_output_refusal_text, ) from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage -from .transformation import LiteLLMAnthropicToResponsesAPIAdapter +from .transformation import ( + REASONING_SUMMARY_PART_SEPARATOR, + LiteLLMAnthropicToResponsesAPIAdapter, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject @@ -29,9 +35,10 @@ class AnthropicResponsesStreamWrapper: response.created -> message_start response.output_item.added -> content_block_start (if message/function_call) response.output_text.delta -> content_block_delta (text_delta) + response.reasoning_summary_part.added -> content_block_delta (thinking_delta separator) response.reasoning_summary_text.delta -> content_block_delta (thinking_delta) response.function_call_arguments.delta -> content_block_delta (input_json_delta) - response.output_item.done -> content_block_stop + response.output_item.done -> content_block_delta (signature_delta) + content_block_stop response.completed -> message_delta + message_stop """ @@ -94,6 +101,38 @@ class AnthropicResponsesStreamWrapper: ) return block_idx + @staticmethod + def _field(source: object, name: str) -> object: + return source.get(name) if isinstance(source, dict) else getattr(source, name, None) + + def _close_reasoning_item(self, item: object, item_id: str | None) -> None: + block_idx: Final = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + encrypted_content: Final = self._field(item, "encrypted_content") + signature: Final = ( + encrypted_reasoning_signature(encrypted_content) + if isinstance(encrypted_content, str) and encrypted_content + else None + ) + if block_idx < 0 and signature is None: + return + if block_idx < 0: + redacted_idx: Final = self._open_block( + item_id, + {"type": "redacted_thinking", "data": signature}, # mutable-ok: API message payload + ) + stop: Final = {"type": "content_block_stop", "index": redacted_idx} # mutable-ok: API message payload + self._chunk_queue.append(stop) + return + if signature is not None: + self._chunk_queue.append( + { # mutable-ok: API message payload + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "signature_delta", "signature": signature}, # mutable-ok: API message payload + } + ) + self._chunk_queue.append({"type": "content_block_stop", "index": block_idx}) # mutable-ok: API message payload + def _process_event(self, event: object) -> None: """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" event_type = getattr(event, "type", None) @@ -175,6 +214,26 @@ class AnthropicResponsesStreamWrapper: ) return + if event_type == "response.reasoning_summary_part.added": + part_item_id: Final = self._field(event, "item_id") + summary_index: Final = self._field(event, "summary_index") + part_block_idx: Final = ( + self._item_id_to_block_index.get(part_item_id, -1) if isinstance(part_item_id, str) else -1 + ) + if part_block_idx < 0 or not isinstance(summary_index, int) or summary_index == 0: + return + self._chunk_queue.append( + { # mutable-ok: API message payload + "type": "content_block_delta", + "index": part_block_idx, + "delta": { # mutable-ok: API message payload + "type": "thinking_delta", + "thinking": REASONING_SUMMARY_PART_SEPARATOR, + }, + } + ) + return + # ---- reasoning summary text delta ---- if event_type == "response.reasoning_summary_text.delta": item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) @@ -220,6 +279,9 @@ class AnthropicResponsesStreamWrapper: item_id = ( getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None ) + if self._field(item, "type") == "reasoning": + self._close_reasoning_item(item, item_id) + return block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index if block_idx < 0: return diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index f1daf2be42a..07419db443c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -13,7 +13,8 @@ from typing import Any, Final, cast from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, - responses_reasoning_item_from_thinking_blocks, + encrypted_reasoning_signature, + responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( @@ -33,6 +34,7 @@ from litellm.types.llms.anthropic import ( AnthropicFinishReason, AnthropicMessagesRequest, AnthropicMessagesToolChoice, + AnthropicResponseContentBlockRedactedThinking, AnthropicResponseContentBlockText, AnthropicResponseContentBlockThinking, AnthropicResponseContentBlockToolUse, @@ -43,11 +45,13 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicUsage, ) from litellm.types.llms.openai import ( - ChatCompletionThinkingBlock, ResponseAPIUsage, ResponsesAPIResponse, ) +REASONING_SUMMARY_PART_SEPARATOR: Final = "\n\n" +RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content" + class LiteLLMAnthropicToResponsesAPIAdapter: """ @@ -163,49 +167,55 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return str(getattr(part, "text", None) or "") @classmethod - def _thinking_blocks_from_reasoning_item( + def _thinking_block_from_reasoning_item( cls, summary: Iterable[object], - ) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload - """Anthropic thinking blocks for one Responses reasoning item. + encrypted_content: object, + ) -> dict[str, Any] | None: # mutable-ok: API message payload + """The one Anthropic block for a Responses reasoning item. - The signature stays empty: only Anthropic can sign a thinking block, and a stand-in - value would be replayed as a real one and rejected by every backend that verifies it. + The item's encrypted reasoning rides the block's opaque field (`signature`, or + `data` when there is no summary text) so the client echoes it back and the next + turn replays the very item OpenAI produced; without it the signature stays empty, + since only Anthropic can sign a thinking block. """ - return tuple( - AnthropicResponseContentBlockThinking( - type="thinking", - thinking=text, - signature=None, - ).model_dump() - for part in summary - if (text := cls._summary_part_text(part)) + text: Final = REASONING_SUMMARY_PART_SEPARATOR.join( + part_text for part in summary if (part_text := cls._summary_part_text(part)) ) + if not isinstance(encrypted_content, str) or not encrypted_content: + if not text: + return None + return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=None).model_dump() + signature: Final = encrypted_reasoning_signature(encrypted_content) + if not text: + return AnthropicResponseContentBlockRedactedThinking(type="redacted_thinking", data=signature).model_dump() + return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=signature).model_dump() @staticmethod def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: """Group a run of consecutive thinking blocks together; keep every other block alone.""" index, block = indexed_block - return "thinking" if block.get("type") == "thinking" else f"block:{index}" + return "thinking" if block.get("type") in ("thinking", "redacted_thinking") else f"block:{index}" @classmethod - def _assistant_group_to_input_item( + def _assistant_group_to_input_items( cls, group: tuple[Mapping[str, object], ...] - ) -> dict[str, Any] | None: # mutable-ok: API message payload + ) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload first: Final = group[0] btype: Final = first.get("type") - if btype == "thinking": - blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group) # cast-ok: untrusted client payload - reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks) - return None if reasoning_item is None else dict(reasoning_item) # mutable-ok: API message payload + if btype in ("thinking", "redacted_thinking"): + replayed: Final = responses_reasoning_items_from_thinking_blocks(group) + return tuple(dict(item) for item in replayed) # mutable-ok: API message payload if btype == "tool_use": - return { # mutable-ok: API message payload - "type": "function_call", - "call_id": first.get("id", ""), - "name": first.get("name", ""), - "arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload - } - return None + return ( + { # mutable-ok: API message payload + "type": "function_call", + "call_id": first.get("id", ""), + "name": first.get("name", ""), + "arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload + }, + ) + return () def translate_messages_to_responses_input( self, @@ -362,7 +372,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: input_items.extend( item for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key) - if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None + for item in self._assistant_group_to_input_items(tuple(block for _, block in group)) ) asst_parts: list[dict[str, Any]] = [ # mutable-ok: API message payload {"type": "output_text", "text": block.get("text", "")} # mutable-ok: API message payload @@ -572,6 +582,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) if reasoning: responses_kwargs["reasoning"] = reasoning + responses_kwargs["include"] = [RESPONSES_INCLUDE_ENCRYPTED_REASONING] # mutable-ok: json list # output_format / output_config.format -> text format # output_format: {"type": "json_schema", "schema": {...}} @@ -634,7 +645,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: for item in response.output: if isinstance(item, ResponseReasoningItem): - content.extend(self._thinking_blocks_from_reasoning_item(item.summary)) + reasoning_block = self._thinking_block_from_reasoning_item(item.summary, item.encrypted_content) + if reasoning_block is not None: + content.append(reasoning_block) elif isinstance(item, ResponseOutputMessage): for part in item.content: @@ -684,11 +697,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ).model_dump() ) elif item_type == "reasoning": - content.extend( - self._thinking_blocks_from_reasoning_item( - cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json - ) + reasoning_block = self._thinking_block_from_reasoning_item( + cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json + item.get("encrypted_content"), ) + if reasoning_block is not None: + content.append(reasoning_block) elif item_type == "function_call": try: input_data = json.loads(item.get("arguments", "{}")) diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index c037f928593..9df124576a0 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -10,10 +10,14 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, add_system_prompt_to_messages, + encrypted_content_from_signature, + encrypted_reasoning_signature, get_file_ids_from_messages, get_format_from_file_id, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, + is_encrypted_reasoning_block, + responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, update_messages_with_model_file_ids, ) @@ -1554,3 +1558,63 @@ class TestRequestContainsImageContent: for _ in range(50): nested = {"type": "tool_result", "content": [nested]} assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False + + +class TestEncryptedReasoningReplay: + """Regression for https://github.com/BerriAI/litellm/issues/40288.""" + + def test_signature_round_trips_the_encrypted_content(self): + assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes" + + @pytest.mark.parametrize("signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7]) + def test_anything_else_is_not_encrypted_content(self, signature): + assert encrypted_content_from_signature(signature) is None + + def test_encrypted_thinking_block_replays_its_own_item(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ( + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Plan."}], "encrypted_content": "gAAAA_1"}, + ) + + def test_encrypted_redacted_block_replays_with_an_empty_summary(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_1"},) + + def test_plain_blocks_collapse_into_one_summary_item_around_encrypted_ones(self): + items = responses_reasoning_items_from_thinking_blocks( + [ + {"type": "thinking", "thinking": "A.", "signature": None}, + {"type": "thinking", "thinking": "B.", "signature": ""}, + {"type": "thinking", "thinking": "C.", "signature": encrypted_reasoning_signature("gAAAA_c")}, + {"type": "redacted_thinking", "data": "anthropic-minted-opaque-data"}, + {"type": "thinking", "thinking": "D."}, + ] + ) + assert items == ( + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}]}, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"}, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]}, + ) + assert all("id" not in item for item in items) + + def test_blocks_without_text_or_encrypted_content_produce_nothing(self): + assert responses_reasoning_items_from_thinking_blocks([{"type": "thinking", "thinking": ""}]) == () + assert responses_reasoning_items_from_thinking_blocks([]) == () + + @pytest.mark.parametrize( + ("block", "expected"), + [ + ({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True), + ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True), + ({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False), + ({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False), + ({"type": "text", "text": encrypted_reasoning_signature("g")}, False), + ("not a block", False), + ], + ) + def test_is_encrypted_reasoning_block(self, block, expected): + assert is_encrypted_reasoning_block(block) is expected diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index dd2d45f00c6..66d10fd1407 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -191,8 +191,16 @@ def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls(): {"type": "thinking", "thinking": "oss reasoning", "signature": None}, {"type": "thinking", "thinking": "oss reasoning", "signature": ""}, {"type": "thinking", "thinking": "oss reasoning"}, + {"type": "thinking", "thinking": "openai reasoning", "signature": "litellm_encrypted_reasoning:gAAAA"}, + {"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"}, + ], + ids=[ + "null_signature", + "empty_signature", + "missing_signature", + "encrypted_reasoning_signature", + "encrypted_reasoning_redacted_data", ], - ids=["null_signature", "empty_signature", "missing_signature"], ) def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): """Open-source reasoning models (DeepSeek-R1, Qwen, etc.) emit thinking blocks @@ -219,7 +227,7 @@ def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): assistant = next(m for m in result if m["role"] == "assistant") content = assistant["content"] assert all( - block.get("type") != "thinking" for block in content + block.get("type") not in ("thinking", "redacted_thinking") for block in content ), f"unsignable thinking block must be dropped, got {content!r}" assert any( block.get("type") == "text" and block.get("text") == "2+2 equals 4." diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index d9df5df426a..bfe2d6b7cea 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -10,6 +10,9 @@ from types import SimpleNamespace sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.experimental_pass_through.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, ) @@ -114,7 +117,7 @@ class TestReasoningItemWithoutSummaryText: """ @staticmethod - def _gpt_turn(reasoning_summary_deltas: list) -> list: + def _gpt_turn(reasoning_summary_deltas: list, encrypted_content: str | None = None) -> list: return [ {"type": "response.created"}, {"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}}, @@ -122,7 +125,10 @@ class TestReasoningItemWithoutSummaryText: {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": delta} for delta in reasoning_summary_deltas ), - {"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}}, + { + "type": "response.output_item.done", + "item": {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted_content}, + }, {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}}, {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "Hello"}, {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}}, @@ -171,6 +177,70 @@ class TestReasoningItemWithoutSummaryText: assert not [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"] +_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read" + + +class TestEncryptedReasoningIsStreamedForReplay: + """Regression for https://github.com/BerriAI/litellm/issues/40288. + + The client echoes a thinking block's signature (or a redacted block's data) back on the + next turn, so the item's ``encrypted_content`` has to reach it through one of those. + """ + + def test_encrypted_content_is_streamed_as_the_signature_before_the_block_closes(self): + chunks = _drain_async( + TestReasoningItemWithoutSummaryText._gpt_turn( + reasoning_summary_deltas=["Weighing options"], encrypted_content=_ENCRYPTED_REASONING + ) + ) + + assert [(c["type"], c.get("index"), c.get("delta", {}).get("type")) for c in chunks[1:5]] == [ + ("content_block_start", 0, None), + ("content_block_delta", 0, "thinking_delta"), + ("content_block_delta", 0, "signature_delta"), + ("content_block_stop", 0, None), + ] + assert chunks[3]["delta"]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING) + + def test_reasoning_without_summary_streams_a_redacted_thinking_block(self): + chunks = _drain_async( + TestReasoningItemWithoutSummaryText._gpt_turn( + reasoning_summary_deltas=[], encrypted_content=_ENCRYPTED_REASONING + ) + ) + + assert [(c["type"], c.get("index")) for c in chunks[1:]] == [ + ("content_block_start", 0), + ("content_block_stop", 0), + ("content_block_start", 1), + ("content_block_delta", 1), + ("content_block_stop", 1), + ] + assert chunks[1]["content_block"] == { + "type": "redacted_thinking", + "data": encrypted_reasoning_signature(_ENCRYPTED_REASONING), + } + + def test_summary_parts_are_separated_inside_the_one_thinking_block(self): + """Two summary parts read as two paragraphs, not as one run-on sentence.""" + events = [ + {"type": "response.created"}, + {"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}}, + {"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 0}, + {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "First."}, + {"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 1}, + {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "Second."}, + {"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}}, + ] + chunks = _process_all(events) + + thinking = "".join( + c["delta"]["thinking"] for c in chunks if c.get("delta", {}).get("type") == "thinking_delta" + ) + assert thinking == "First.\n\nSecond." + assert [c["type"] for c in chunks].count("content_block_start") == 1 + + class TestToolUseBlockClosedExactlyOnce: """Regression for https://github.com/BerriAI/litellm/issues/37273. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 9f8414afa38..79eaf6b60f2 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -19,6 +19,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, + encrypted_reasoning_signature, ) from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, @@ -566,6 +567,66 @@ class TestTranslateMessagesToResponsesInput: result = _translate_messages(messages) assert "id" not in result[0] + def test_thinking_block_with_encrypted_signature_replays_the_encrypted_content(self): + """Regression for https://github.com/BerriAI/litellm/issues/40288 (inbound fault site).""" + messages = [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "Private reasoning.", + "signature": encrypted_reasoning_signature("gAAAA_turn_one"), + } + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Private reasoning."}], + "encrypted_content": "gAAAA_turn_one", + } + ] + + def test_redacted_thinking_with_encrypted_data_replays_the_encrypted_content(self): + messages = [ + { + "role": "assistant", + "content": [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_turn_one")}], + } + ] + result = _translate_messages(messages) + assert result == [{"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_turn_one"}] + + def test_each_encrypted_thinking_block_stays_its_own_reasoning_item(self): + """Two upstream items must not be merged into one, or the encrypted content of one is lost.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "First.", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "thinking", "thinking": "Second.", "signature": encrypted_reasoning_signature("gAAAA_2")}, + ], + } + ] + result = _translate_messages(messages) + assert [item["encrypted_content"] for item in result] == ["gAAAA_1", "gAAAA_2"] + + def test_anthropic_signed_thinking_block_replays_as_a_summary_only_item(self): + """A real Anthropic signature is opaque here, so it never masquerades as encrypted content.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": "Private reasoning.", "signature": "ErcBCkgIValid"}], + } + ] + result = _translate_messages(messages) + assert result == [ + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Private reasoning."}]} + ] + def test_consecutive_thinking_blocks_become_one_reasoning_item(self): """Summary parts of one upstream reasoning item are regrouped into that item.""" messages = [ @@ -1101,6 +1162,13 @@ class TestTranslateRequestBroaderCoverage: req = _make_request(thinking={"type": "disabled"}) kwargs = _ADAPTER.translate_request(req) assert "reasoning" not in kwargs + assert "include" not in kwargs + + def test_thinking_asks_for_the_encrypted_reasoning(self): + """The documented way to get reasoning that survives store=false is to ask for it.""" + req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["include"] == ["reasoning.encrypted_content"] def test_metadata_user_id_mapped_to_user(self): req = _make_request(metadata={"user_id": "user-42"}) @@ -1229,7 +1297,9 @@ def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMo return item -def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> MagicMock: +def _make_reasoning_item( + summaries: List[str], item_id: str = "rs_test_1", encrypted_content: str | None = None +) -> MagicMock: """Build a mock ResponseReasoningItem.""" from openai.types.responses import ResponseReasoningItem # type: ignore[import] @@ -1242,9 +1312,13 @@ def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> Ma item = MagicMock(spec=ResponseReasoningItem) item.id = item_id item.summary = summary_mocks + item.encrypted_content = encrypted_content return item +_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read" + + class TestTranslateResponse: """Responses API -> AnthropicMessagesResponse conversion.""" @@ -1369,7 +1443,81 @@ class TestTranslateResponse: reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123") response = _make_mock_response(output=[reasoning]) result: Any = _ADAPTER.translate_response(response) - assert [block["signature"] for block in result["content"]] == [None, None] + assert [block["signature"] for block in result["content"]] == [None] + assert "rs_abc123" not in json.dumps(result["content"]) + + def test_summary_parts_join_into_one_thinking_block(self): + """One reasoning item is one block, so its signature is echoed back exactly once.""" + reasoning = _make_reasoning_item(["Part one.", "Part two."]) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert [block["thinking"] for block in result["content"]] == ["Part one.\n\nPart two."] + + def test_encrypted_content_rides_the_thinking_signature(self): + """Regression for https://github.com/BerriAI/litellm/issues/40288 (outbound fault site).""" + reasoning = _make_reasoning_item(["Part one."], item_id="rs_abc123", encrypted_content=_ENCRYPTED_REASONING) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [ + { + "type": "thinking", + "thinking": "Part one.", + "signature": encrypted_reasoning_signature(_ENCRYPTED_REASONING), + } + ] + + def test_reasoning_without_summary_becomes_redacted_thinking(self): + """With summaries off the encrypted reasoning still has to reach the client to be replayed.""" + reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature(_ENCRYPTED_REASONING)} + ] + + def test_dict_reasoning_item_carries_its_encrypted_content(self): + response = _make_mock_response( + output=[ + { + "type": "reasoning", + "id": "rs_dict_1", + "encrypted_content": _ENCRYPTED_REASONING, + "summary": [{"type": "summary_text", "text": "Weighing the options."}], + } + ] + ) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING) + + def test_reasoning_item_round_trip_is_byte_stable(self): + """Regression for https://github.com/BerriAI/litellm/issues/40288. + + The reasoning item the next turn replays must be the one OpenAI produced, with its + encrypted reasoning intact, and identical on every later turn so the prompt cache + prefix keeps matching. + """ + reasoning = _make_reasoning_item(["Part one.", "Part two."], encrypted_content=_ENCRYPTED_REASONING) + turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning])) + history = [{"role": "assistant", "content": turn["content"]}] + + replayed_items = [_translate_messages(history) for _ in range(2)] + + assert replayed_items[0] == replayed_items[1] + assert replayed_items[0] == [ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Part one.\n\nPart two."}], + "encrypted_content": _ENCRYPTED_REASONING, + } + ] + + def test_redacted_reasoning_round_trip_replays_the_encrypted_content(self): + reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING) + turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning])) + + replayed = _translate_messages([{"role": "assistant", "content": turn["content"]}]) + + assert replayed == [{"type": "reasoning", "summary": [], "encrypted_content": _ENCRYPTED_REASONING}] def test_dict_reasoning_item_becomes_thinking_block(self): """A reasoning item arriving as a plain dict is kept, not dropped.""" @@ -1385,14 +1533,26 @@ class TestTranslateResponse: result: Any = _ADAPTER.translate_response(response) assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}] - def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self): + @pytest.mark.parametrize( + ("summaries", "encrypted_content"), + [ + (["Part one."], None), + (["Part one."], _ENCRYPTED_REASONING), + ([], _ENCRYPTED_REASONING), + ], + ids=["unsigned_thinking", "encrypted_thinking", "encrypted_redacted_thinking"], + ) + def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self, summaries, encrypted_content): """Replaying this turn to an Anthropic model must not send a signature it cannot verify.""" from litellm.litellm_core_utils.prompt_templates.factory import ( _drop_unsignable_thinking_blocks, ) - response = _make_mock_response(output=[_make_reasoning_item(["Part one."], item_id="rs_abc123")]) + response = _make_mock_response( + output=[_make_reasoning_item(summaries, item_id="rs_abc123", encrypted_content=encrypted_content)] + ) result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 assert _drop_unsignable_thinking_blocks(result["content"]) == [] def test_usage_mapped_correctly(self): diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index ae620fdd6dc..336cbf57e29 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1541,6 +1541,30 @@ class TestAnthropicThinkingSignatureSelfHeal: out = strip_empty_content_blocks_from_anthropic_messages(msgs) assert [b["type"] for b in out[0]["content"]] == ["thinking"] + def test_strip_drops_encrypted_reasoning_blocks_from_the_responses_bridge(self): + """A session resumed on an Anthropic model replays reasoning only OpenAI can verify.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, + ) + from litellm.llms.anthropic.common_utils import ( + strip_empty_content_blocks_from_anthropic_messages, + ) + + msgs = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."}, + ], + } + ] + out = strip_empty_content_blocks_from_anthropic_messages(msgs) + assert [b["type"] for b in out[0]["content"]] == ["redacted_thinking", "text"] + assert len(msgs[0]["content"]) == 4 + def test_strip_empty_text_blocks_treats_null_text_as_empty(self): from litellm.llms.anthropic.common_utils import ( strip_empty_content_blocks_from_anthropic_messages, From dd707da81c8e7ec9611da406c569b61b4e5385bc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:28:58 -0700 Subject: [PATCH 168/411] fix(bedrock): forward raw reasoning_effort to DeepSeek V3, drop it only for R1 --- .../bedrock/chat/converse_transformation.py | 52 ++++++----- .../chat/test_converse_transformation.py | 89 +++++++++++++++---- 2 files changed, 103 insertions(+), 38 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6fe69b012ca..a9f34cdd683 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -422,13 +422,13 @@ class AmazonConverseConfig(BaseConfig): """ Handle the reasoning_effort parameter based on the model type. - - GPT-OSS models: passed through unchanged via additionalModelRequestFields. + - GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields. - OpenAI GPT-5.x models: mapped to ``reasoning.effort`` via additionalModelRequestFields. - Nova 2 models: transformed to reasoningConfig. - Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on adaptive Claude 4.6 / 4.7). """ - if "gpt-oss" in model: + if "gpt-oss" in model or "deepseek" in model: optional_params["reasoning_effort"] = reasoning_effort elif "openai.gpt-5" in model: reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort} @@ -509,14 +509,19 @@ class AmazonConverseConfig(BaseConfig): ) thinking["budget_tokens"] = BEDROCK_MIN_THINKING_BUDGET_TOKENS - def _model_accepts_anthropic_thinking_param(self, model: str, base_model: str) -> bool: - """Whether the model accepts the Anthropic-shaped ``thinking`` / ``reasoning_effort`` request field. + def _is_deepseek_model(self, model: str, base_model: str) -> bool: + return "deepseek" in model or "deepseek" in base_model - The Converse mapping serializes ``thinking`` into ``additionalModelRequestFields`` in Anthropic's - shape. Only Anthropic Claude reasoning models accept that field; DeepSeek models reason natively - and reject it (a 400 when it leaks through), even though they advertise ``supports_reasoning``. + def _is_deepseek_r1_model(self, model: str, base_model: str) -> bool: + return "deepseek.r1" in model or "deepseek.r1" in base_model + + def _model_accepts_anthropic_thinking_param(self, model: str, base_model: str) -> bool: + """Whether the model accepts the Anthropic-shaped ``thinking`` request field. + + Only Claude reasoning models accept it. DeepSeek advertises ``supports_reasoning`` but reasons + natively: R1 returns a 400 when the field is sent and V3 silently ignores it. """ - if "deepseek" in model or "deepseek" in base_model: + if self._is_deepseek_model(model=model, base_model=base_model): return False return ( "claude-3-7" in model @@ -526,17 +531,13 @@ class AmazonConverseConfig(BaseConfig): or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) ) - def _model_reasons_natively_and_rejects_request_param(self, model: str, base_model: str) -> bool: - """Whether the model reasons natively and rejects any reasoning request field on Converse. + def _model_rejects_reasoning_effort_param(self, model: str, base_model: str) -> bool: + """Whether the model returns a 400 for every ``reasoning_effort`` shape on Converse. - The Converse mapping serializes ``thinking`` into ``additionalModelRequestFields`` in Anthropic's - shape and ``reasoning_effort`` into a provider-specific shape. DeepSeek reasons on its own and - returns a 400 when either field is sent, even though it advertises ``supports_reasoning``, so both - must be dropped for it. Every other model either accepts one of those shapes (Claude ``thinking``, - gpt-oss / Nova 2 ``reasoning_effort``) or is an opaque ARN we can't introspect, so we leave those - untouched rather than silently degrading reasoning. + DeepSeek R1 always reasons and rejects any reasoning request field. DeepSeek V3 accepts a raw + ``reasoning_effort`` like gpt-oss does, and every other model maps it to a shape it accepts. """ - return "deepseek" in model or "deepseek" in base_model + return self._is_deepseek_r1_model(model=model, base_model=base_model) def get_supported_openai_params(self, model: str) -> list[str]: from litellm.utils import supports_function_calling @@ -595,6 +596,9 @@ class AmazonConverseConfig(BaseConfig): if "gpt-oss" in model or "openai.gpt-5" in model or "openai.gpt-5" in base_model: supported_params.append("reasoning_effort") + elif self._is_deepseek_model(model=model, base_model=base_model): + if not self._is_deepseek_r1_model(model=model, base_model=base_model): + supported_params.append("reasoning_effort") elif self._is_nova_2_model(model): # Nova 2 models support reasoning_effort (transformed to reasoningConfig) # These models use a different reasoning structure than Anthropic's thinking parameter @@ -891,8 +895,10 @@ class AmazonConverseConfig(BaseConfig): drop_params: bool, ) -> dict: is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params) - drop_reasoning_request_param: Final = self._model_reasons_natively_and_rejects_request_param( - model=model, base_model=BedrockModelInfo.get_base_model(model) + base_model: Final = BedrockModelInfo.get_base_model(model) + drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model) + drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param( + model=model, base_model=base_model ) for param, value in non_default_params.items(): @@ -942,9 +948,9 @@ class AmazonConverseConfig(BaseConfig): optional_params["_parallel_tool_use_config"] = { "tool_choice": {"type": "auto", "disable_parallel_tool_use": not value} } - if param == "thinking" and drop_reasoning_request_param: + if param == "thinking" and drop_thinking_param: verbose_logger.debug( - "Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively and rejects it.", + "Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively.", model, ) elif param == "thinking" and "openai.gpt-5" not in model: @@ -973,9 +979,9 @@ class AmazonConverseConfig(BaseConfig): AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( model=model, optional_params=optional_params, custom_llm_provider="bedrock" ) - elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_request_param: + elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param: verbose_logger.debug( - "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it reasons natively and rejects it.", + "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.", model, ) elif param == "reasoning_effort" and isinstance(value, str): diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 2fa32cd2037..e336cb8a14a 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -873,31 +873,55 @@ def test_get_supported_openai_params(): ], ) def test_bedrock_deepseek_does_not_advertise_thinking(model): - """DeepSeek reasons natively on Bedrock and rejects the Anthropic-shaped - `thinking`/`reasoning_effort` field, so it must not be advertised as supported - (otherwise it leaks into additionalModelRequestFields and Bedrock 400s).""" + """DeepSeek reasons natively on Bedrock and does not take the Anthropic-shaped `thinking` + field (R1 400s on it, V3 ignores it), so it must not be advertised as supported.""" config = AmazonConverseConfig() supported_params = config.get_supported_openai_params(model=model) assert "thinking" not in supported_params - assert "reasoning_effort" not in supported_params + assert "output_config" not in supported_params -def test_bedrock_deepseek_r1_thinking_raises_without_drop_params(): - """Passing `thinking` to Bedrock DeepSeek R1 must fail client-side with a clear - UnsupportedParamsError instead of leaking through and hitting a Bedrock 400.""" +@pytest.mark.parametrize("model", ["bedrock/us.deepseek.r1-v1:0", "bedrock/converse/us.deepseek.r1-v1:0"]) +def test_bedrock_deepseek_r1_does_not_advertise_reasoning_effort(model): + """DeepSeek R1 always reasons and returns a 400 for any reasoning_effort shape.""" + config = AmazonConverseConfig() + assert "reasoning_effort" not in config.get_supported_openai_params(model=model) + + +@pytest.mark.parametrize("model", ["bedrock/deepseek.v3-v1:0", "bedrock/deepseek.v3.2", "bedrock/us.deepseek.v3.2"]) +def test_bedrock_deepseek_v3_advertises_reasoning_effort(model): + """DeepSeek V3 on Bedrock accepts a raw reasoning_effort in additionalModelRequestFields.""" + config = AmazonConverseConfig() + assert "reasoning_effort" in config.get_supported_openai_params(model=model) + + +@pytest.mark.parametrize("model", ["us.deepseek.r1-v1:0", "deepseek.v3.2"]) +def test_bedrock_deepseek_thinking_raises_without_drop_params(model): + """Passing `thinking` to Bedrock DeepSeek must fail client-side with a clear + UnsupportedParamsError instead of leaking through to Bedrock.""" with pytest.raises(litellm.UnsupportedParamsError): litellm.utils.get_optional_params( - model="us.deepseek.r1-v1:0", + model=model, custom_llm_provider="bedrock", thinking={"type": "enabled", "budget_tokens": 1024}, ) -def test_bedrock_deepseek_r1_thinking_dropped_does_not_leak_into_request(): +def test_bedrock_deepseek_r1_reasoning_effort_raises_without_drop_params(): + with pytest.raises(litellm.UnsupportedParamsError): + litellm.utils.get_optional_params( + model="us.deepseek.r1-v1:0", + custom_llm_provider="bedrock", + reasoning_effort="high", + ) + + +@pytest.mark.parametrize("model", ["us.deepseek.r1-v1:0", "deepseek.v3.2"]) +def test_bedrock_deepseek_thinking_dropped_does_not_leak_into_request(model): """With drop_params, `thinking` is dropped rather than forwarded into - additionalModelRequestFields for Bedrock DeepSeek R1.""" + additionalModelRequestFields for Bedrock DeepSeek.""" optional_params = litellm.utils.get_optional_params( - model="us.deepseek.r1-v1:0", + model=model, custom_llm_provider="bedrock", thinking={"type": "enabled", "budget_tokens": 1024}, drop_params=True, @@ -906,20 +930,20 @@ def test_bedrock_deepseek_r1_thinking_dropped_does_not_leak_into_request(): config = AmazonConverseConfig() request = config._transform_request( - model="bedrock/converse/us.deepseek.r1-v1:0", + model=f"bedrock/converse/{model}", messages=[{"role": "user", "content": "Say hi in one word."}], optional_params=optional_params, litellm_params={}, headers={}, ) - assert "thinking" not in request.get("additionalModelRequestFields", {}) + assert "thinking" not in (request.get("additionalModelRequestFields") or {}) @pytest.mark.parametrize("param", ["thinking", "reasoning_effort"]) def test_bedrock_deepseek_r1_reasoning_params_not_forwarded_by_map(param): """Even when map_openai_params is called directly (bypassing the supported-params - gate), DeepSeek R1 must not forward the Anthropic-shaped thinking/reasoning_effort - into additionalModelRequestFields, since Bedrock rejects it with a 400.""" + gate), DeepSeek R1 must not forward thinking/reasoning_effort into + additionalModelRequestFields, since Bedrock rejects both with a 400.""" config = AmazonConverseConfig() model = "bedrock/converse/us.deepseek.r1-v1:0" value = {"type": "enabled", "budget_tokens": 1024} if param == "thinking" else "high" @@ -943,6 +967,41 @@ def test_bedrock_deepseek_r1_reasoning_params_not_forwarded_by_map(param): assert request.get("additionalModelRequestFields") is None +def test_bedrock_deepseek_v3_reasoning_effort_forwarded_raw(): + """DeepSeek V3 takes reasoning_effort verbatim in additionalModelRequestFields, never + converted into the Anthropic `thinking` block that Claude models get.""" + config = AmazonConverseConfig() + model = "bedrock/deepseek.v3.2" + optional_params = config.map_openai_params( + non_default_params={"reasoning_effort": "high", "max_tokens": 100}, + optional_params={}, + model=model, + drop_params=False, + ) + assert "thinking" not in optional_params + + request = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Say hi in one word."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert request["additionalModelRequestFields"] == {"reasoning_effort": "high"} + + +def test_bedrock_deepseek_v3_thinking_dropped_by_map(): + config = AmazonConverseConfig() + optional_params = config.map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 1024}, "max_tokens": 100}, + optional_params={}, + model="bedrock/deepseek.v3.2", + drop_params=False, + ) + assert "thinking" not in optional_params + assert "reasoning_effort" not in optional_params + + @pytest.mark.parametrize( "model, param, value, kept_key", [ From 8d040d89e68c075b105a6369e5d95fdc7c1ba8f0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:39:56 -0700 Subject: [PATCH 169/411] fix(policy_engine): keep legacy hooks off streams their route cannot assemble and off guardrails with their own iterator hook The streaming pipeline step only takes a post-call hook on routes whose translation assembles the streamed response (chat completions, Responses, Messages). On /v1/completions, the Gemini streamGenerateContent route, and A2A streams the pipeline is skipped with the merge-base warning and the hook runs on its own afterwards, instead of getting a None response while the header says the guardrail ran. A guardrail that overrides async_post_call_streaming_iterator_hook next to its post-call hook keeps its native per-chunk path rather than running buffered through the adapter --- .../chat/guardrail_translation/handler.py | 1 + .../guardrail_translation/base_translation.py | 7 ++ .../chat/guardrail_translation/handler.py | 1 + .../guardrail_translation/handler.py | 1 + .../proxy/policy_engine/pipeline_executor.py | 14 ++-- litellm/proxy/utils.py | 60 +++++++++++---- .../policy_engine/test_pipeline_executor.py | 46 +++-------- .../proxy_logging/test_guardrail_pipeline.py | 77 ++++++++++++++++++- 8 files changed, 148 insertions(+), 59 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index f61cfe58e80..b83235f660f 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -171,6 +171,7 @@ class AnthropicMessagesHandler(BaseTranslation): """ delivers_ended_stream_text_rewrites = True + assembles_streamed_response = True def __init__(self): super().__init__() diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index bef472b2882..375b22ff5f6 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -60,6 +60,13 @@ class BaseTranslation(ABC): text rewrites on every other translation, are undeliverable: the pipeline executor discards them and releases the original chunks.""" + assembles_streamed_response: ClassVar[bool] = False + """Whether ``process_output_streaming_response`` stores the assembled response of an + ended stream under ``request_data["response"]`` before scanning it, the way the chat, + Responses, and Messages translations do. A streaming pipeline runs a guardrail that only + has the legacy post-call hook against that response, so on a translation without it such + a guardrail keeps running on its own.""" + def post_call_hook_response(self, response: object) -> object: """The ``response`` this endpoint's non-streaming post-call hooks receive, derived from the object the translation stores under ``request_data["response"]`` while scanning an diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 80292aef2cf..b9b88061fec 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -79,6 +79,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): """ delivers_ended_stream_text_rewrites = True + assembles_streamed_response = True def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: """ diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index b0f79552bc5..e0ba80706df 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -341,6 +341,7 @@ class OpenAIResponsesHandler(BaseTranslation): """ delivers_ended_stream_text_rewrites = True + assembles_streamed_response = True def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: """ diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 52985d49c7e..ab5ad01fda9 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -556,12 +556,14 @@ class PipelineExecutor: @staticmethod def supports_streaming_execution(callback: CustomGuardrail) -> bool: """Whether a streaming pipeline step can run this guardrail against the buffered - stream: through the unified path, or through its own post-call hook on the - assembled response. A guardrail with neither (one that only rewrites the stream - through its iterator hook) has to keep running on its own.""" - return ( - PipelineExecutor.supports_unified_execution(callback) - or type(callback).async_post_call_success_hook is not CustomLogger.async_post_call_success_hook + stream: through the unified path, or through its post-call hook on the assembled + response when that hook is its only streaming path. A guardrail with its own + streaming iterator hook, or with neither hook, keeps running on its own.""" + callback_type: Final = type(callback) + return PipelineExecutor.supports_unified_execution(callback) or ( + callback_type.async_post_call_success_hook is not CustomLogger.async_post_call_success_hook + and callback_type.async_post_call_streaming_iterator_hook + is CustomLogger.async_post_call_streaming_iterator_hook ) @staticmethod diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 23260785b97..67a579c4d15 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -192,6 +192,7 @@ if TYPE_CHECKING: from prisma.types import HttpConfig from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.models.team import LiteLLM_TeamTableCachedObj from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction @@ -517,9 +518,17 @@ def _merge_pipeline_metadata_writes( _merge_pipeline_metadata_bucket(data, bucket_key, modified_data.get(bucket_key)) -def _pipeline_step_supports_streaming(guardrail_name: str) -> bool: +def _pipeline_step_supports_streaming(guardrail_name: str, translation: "BaseTranslation | None") -> bool: callback: Final = PipelineExecutor.find_guardrail_callback(guardrail_name) - return callback is not None and PipelineExecutor.supports_streaming_execution(callback) + if callback is None: + return False + if PipelineExecutor.supports_unified_execution(callback): + return True + return ( + translation is not None + and type(translation).assembles_streamed_response + and PipelineExecutor.supports_streaming_execution(callback) + ) def _post_call_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "GuardrailPipeline"], ...]: @@ -541,42 +550,57 @@ def _warn_background_skips_post_call_pipelines(data: Mapping[str, object]) -> No ) -def _pipeline_unsupported_streaming_guardrails(pipeline: "GuardrailPipeline") -> tuple[str, ...]: +def _pipeline_unsupported_streaming_guardrails( + pipeline: "GuardrailPipeline", translation: "BaseTranslation | None" +) -> tuple[str, ...]: return tuple( dict.fromkeys( - step.guardrail for step in pipeline.steps if not _pipeline_step_supports_streaming(step.guardrail) + step.guardrail + for step in pipeline.steps + if not _pipeline_step_supports_streaming(step.guardrail, translation) ) ) -def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: - unsupported: Final = _pipeline_unsupported_streaming_guardrails(pipeline) +def _pipeline_is_streamable( + policy_name: str, pipeline: "GuardrailPipeline", translation: "BaseTranslation | None" +) -> bool: + unsupported: Final = _pipeline_unsupported_streaming_guardrails(pipeline, translation) if not unsupported: return True verbose_proxy_logger.warning( - "Policy '%s' has post_call pipeline guardrails with neither the unified apply_guardrail interface nor a " - "post-call hook, one of which streaming pipelines need; the stream skips the pipeline and its guardrails " - "run on their own: %s", + "Policy '%s' has post_call pipeline guardrails a streaming pipeline cannot run on this route yet; they " + "need the unified apply_guardrail interface, or a post-call hook without a streaming iterator hook on a " + "route whose translation assembles the streamed response. The stream skips the pipeline and its " + "guardrails run on their own: %s", policy_name, ", ".join(unsupported), ) return False -def _route_supports_streaming_pipelines(user_api_key_dict: UserAPIKeyAuth) -> bool: - return not user_api_key_dict.request_route or resolve_endpoint_translation(user_api_key_dict, None) is not None +def _streaming_pipeline_translation(user_api_key_dict: UserAPIKeyAuth) -> "BaseTranslation | None": + resolved: Final = resolve_endpoint_translation(user_api_key_dict, None) + return None if resolved is None else resolved[1] + + +def _route_supports_streaming_pipelines( + user_api_key_dict: UserAPIKeyAuth, translation: "BaseTranslation | None" +) -> bool: + return not user_api_key_dict.request_route or translation is not None def stream_gated_guardrail_names( request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth ) -> frozenset[str]: - if not _route_supports_streaming_pipelines(user_api_key_dict): + translation: Final = _streaming_pipeline_translation(user_api_key_dict) + if not _route_supports_streaming_pipelines(user_api_key_dict, translation): return frozenset() return _pipeline_step_guardrail_names( tuple( (policy_name, pipeline) for policy_name, pipeline in _post_call_pipelines(request_data) - if not _pipeline_unsupported_streaming_guardrails(pipeline) + if not _pipeline_unsupported_streaming_guardrails(pipeline, translation) ) ) @@ -589,8 +613,9 @@ def _streamable_post_call_pipelines( Streaming pipelines scan the buffered stream through the endpoint guardrail translation of the request route, so every step's guardrail needs either the - unified apply_guardrail interface or a post-call hook to run against the - assembled response, and the route needs a translation. A pipeline that + unified apply_guardrail interface or, on a route whose translation assembles + the streamed response, a post-call hook that is its only streaming path, and + the route needs a translation. A pipeline that cannot be run that way yet is left out and its guardrails run on the stream on their own, the way they did before pipelines ran on streams at all, with a warning naming the pipeline. @@ -598,7 +623,8 @@ def _streamable_post_call_pipelines( post_call_pipelines: Final = _post_call_pipelines(request_data) if not post_call_pipelines: return () - if not _route_supports_streaming_pipelines(user_api_key_dict): + translation: Final = _streaming_pipeline_translation(user_api_key_dict) + if not _route_supports_streaming_pipelines(user_api_key_dict, translation): verbose_proxy_logger.warning( "Policies with post_call guardrail pipelines cannot scan streaming responses on route %s yet " "(no endpoint guardrail translation); the stream skips the pipelines and their guardrails run " @@ -610,7 +636,7 @@ def _streamable_post_call_pipelines( return tuple( (policy_name, pipeline) for policy_name, pipeline in post_call_pipelines - if _pipeline_is_streamable(policy_name, pipeline) + if _pipeline_is_streamable(policy_name, pipeline, translation) ) diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 6de7f7b593d..e2523567065 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -1589,28 +1589,14 @@ async def test_streaming_step_discards_a_legacy_tool_call_rewrite_on_a_tool_only assert chunks == [_tool_only_chunk()] -class _ResponselessLegacyScanningTranslation(_LegacyScanningTranslation): - """Like a handler that never stores the assembled response under request_data["response"].""" +class _NoHooksGuardrail(CustomGuardrail): + pass - def post_call_hook_response(self, response): - return response - async def process_output_streaming_response( - self, - responses_so_far, - guardrail_to_apply, - litellm_logging_obj=None, - user_api_key_dict=None, - request_data=None, - deliver_ended_stream_rewrites=False, - ): - await guardrail_to_apply.apply_guardrail( - inputs={"texts": [responses_so_far[0]["text"]]}, - request_data=request_data, - input_type="response", - logging_obj=litellm_logging_obj, - ) - return responses_so_far +class _IteratorAndLegacyHookGuardrail(_LegacyHookGuardrail): + async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): + async for item in response: + yield item class _UnscannableRewriteTranslation(_LegacyScanningTranslation): @@ -1623,21 +1609,11 @@ class _UnscannableRewriteTranslation(_LegacyScanningTranslation): return response -@pytest.mark.asyncio -async def test_streaming_step_leaves_the_stream_alone_when_the_hook_gets_no_response(monkeypatch, caplog): - guardrail = _LegacyHookGuardrail() - chunks = [_chunk()] - - result = await _run_legacy_streaming_step( - monkeypatch, guardrail, chunks, translation=_ResponselessLegacyScanningTranslation() - ) - - assert result.terminal_action == "allow" - assert [step.outcome for step in result.step_results] == ["pass"] - assert guardrail.calls[0]["response"] is None - assert chunks == [_chunk()] - assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"] - assert not any("discarded" in record.getMessage() for record in caplog.records) +def test_streaming_execution_runs_legacy_hooks_only_when_that_hook_is_their_only_streaming_path(): + assert PipelineExecutor.supports_streaming_execution(_LegacyHookGuardrail()) is True + assert PipelineExecutor.supports_streaming_execution(_NativeHooksGuardrail()) is True + assert PipelineExecutor.supports_streaming_execution(_IteratorAndLegacyHookGuardrail()) is False + assert PipelineExecutor.supports_streaming_execution(_NoHooksGuardrail(guardrail_name="neither")) is False @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 46d6f2bb7a9..ac303e8a027 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -28,7 +28,7 @@ from litellm.integrations.custom_guardrail import ( from litellm.integrations.prometheus import PrometheusLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header -from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines +from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines, stream_gated_guardrail_names from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks from litellm.types.proxy.policy_engine.pipeline_types import ( @@ -1539,6 +1539,21 @@ def _iterator_hook_only_guardrail(name: str, seen: Dict[str, Any]) -> CustomGuar return IteratorHookGuardrail(guardrail_name=name, event_hook=GuardrailEventHooks.post_call, default_on=True) +def _iterator_and_legacy_hook_guardrail(name: str, seen: Dict[str, Any]) -> CustomGuardrail: + class IteratorAndLegacyHookGuardrail(CustomGuardrail): + async def async_post_call_success_hook(self, data, user_api_key_dict, response): + seen["success_hook_calls"] = seen.get("success_hook_calls", 0) + 1 + return None + + async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): + seen["iterator_hook_calls"] = seen.get("iterator_hook_calls", 0) + 1 + async for item in response: + item.choices[0].delta.content = f"[governed] {item.choices[0].delta.content}" + yield item + + return IteratorAndLegacyHookGuardrail(guardrail_name=name, event_hook=GuardrailEventHooks.post_call, default_on=True) + + def _rewritten_model_response(response: Any) -> litellm.ModelResponse: payload = response.model_dump() payload["choices"][0]["message"]["content"] = "[REWRITTEN] " + payload["choices"][0]["message"]["content"] @@ -1572,6 +1587,42 @@ def test_streamable_post_call_pipelines_keeps_hook_guardrails_and_drops_iterator assert not any("'governed'" in message or "gr-legacy" in message for message in _warnings(caplog)) +@pytest.mark.parametrize( + "request_route", + [None, "/v1/completions", "/v1beta/models/gemini-2.5-flash:streamGenerateContent", "/a2a/agent"], +) +def test_streamable_post_call_pipelines_keeps_legacy_hooks_off_routes_that_assemble_no_response( + make_user_api_key_auth, monkeypatch, caplog, request_route +): + monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail({})]) + legacy = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")]) + data = {"metadata": {"_guardrail_pipelines": [("legacy-governance", legacy)]}} + auth = make_user_api_key_auth(request_route=request_route) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + streamable = _streamable_post_call_pipelines(data, auth) + + assert streamable == () + assert stream_gated_guardrail_names(data, auth) == frozenset() + assert any("'legacy-governance'" in message and "gr-post" in message for message in _warnings(caplog)) + + +def test_streamable_post_call_pipelines_keeps_guardrails_with_their_own_iterator_hook_on_their_own_path( + make_user_api_key_auth, monkeypatch, caplog +): + monkeypatch.setattr(litellm, "callbacks", [_iterator_and_legacy_hook_guardrail("gr-post", {})]) + both_hooks = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")]) + data = {"metadata": {"_guardrail_pipelines": [("both-hooks", both_hooks)]}} + auth = make_user_api_key_auth(request_route="/v1/chat/completions") + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + streamable = _streamable_post_call_pipelines(data, auth) + + assert streamable == () + assert stream_gated_guardrail_names(data, auth) == frozenset() + assert any("'both-hooks'" in message and "gr-post" in message for message in _warnings(caplog)) + + def test_streamable_post_call_pipelines_is_empty_on_route_without_translation( make_user_api_key_auth, monkeypatch, caplog ): @@ -1762,6 +1813,30 @@ async def test_streaming_iterator_hook_runs_iterator_hook_guardrail_whose_pipeli assert any("'response-governance'" in message and "gr-post" in message for message in _warnings(caplog)) +@pytest.mark.asyncio +async def test_streaming_iterator_hook_runs_the_iterator_hook_of_a_guardrail_that_also_has_a_post_call_hook( + proxy_logging, make_user_api_key_auth, monkeypatch, caplog +): + seen: Dict[str, Any] = {} + monkeypatch.setattr(litellm, "callbacks", [_iterator_and_legacy_hook_guardrail("gr-post", seen)]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _post_call_pipeline_data(stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + delivered = [ + item + async for item in proxy_logging.async_post_call_streaming_iterator_hook( + user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"), + response=_async_chunk_iter(_stream_chunks()), + request_data=data, + ) + ] + + assert seen == {"iterator_hook_calls": 1} + assert [item.choices[0].delta.content for item in delivered] == ["[governed] hello ", "[governed] world"] + assert any("'response-governance'" in message and "gr-post" in message for message in _warnings(caplog)) + + @pytest.mark.asyncio @pytest.mark.parametrize( "rewrite_attribute, value", From 9a721abf0d098f40caa6ba67d343e99de74f7fc2 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 03:50:49 +0000 Subject: [PATCH 170/411] test(cost-map): clear LITELLM_LOCAL_MODEL_COST_MAP in register_model url test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index e90372141bd..e42608c9904 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2386,6 +2386,7 @@ def test_register_model_with_scientific_notation(): @respx.mock def test_register_model_url_fetch_uses_single_attempt(monkeypatch): + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) before = dict(litellm.model_cost) threads_before = {thread.name for thread in threading.enumerate()} From 15392e7b3a298bec57f4aae31e863e39fb182cc6 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 21:26:04 -0700 Subject: [PATCH 171/411] fix(mcp): surface connection test failures safely --- litellm/experimental_mcp_client/client.py | 21 +++ .../mcp_server/rest_endpoints.py | 59 ++++++- .../test_mcp_client.py | 124 ++++++++++++++- .../mcp_server/test_rest_endpoints.py | 148 +++++++++++++++++- 4 files changed, 334 insertions(+), 18 deletions(-) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 3503468c735..cabcc6c03ba 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -18,6 +18,7 @@ from mcp import ClientSession, McpError, ReadResourceResult, Resource, StdioServ from mcp.client.sse import sse_client from mcp.client.stdio import stdio_client from mcp.shared.message import SessionMessage +from mcp.shared.session import RequestResponder from typing_extensions import Unpack _TransportStreams: TypeAlias = tuple[ @@ -56,10 +57,13 @@ def missing_streamable_http_client_error() -> ImportError: from mcp.types import CallToolRequestParams as MCPCallToolRequestParams from mcp.types import CallToolResult as MCPCallToolResult from mcp.types import ( + ClientResult, GetPromptRequestParams, GetPromptResult, Prompt, ResourceTemplate, + ServerNotification, + ServerRequest, TextContent, ) from mcp.types import Tool as MCPTool @@ -442,6 +446,18 @@ class MCPClient: in_flight_error: BaseException | None = None try: read_stream, write_stream = transport[0], transport[1] + stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future() + + async def receive_message( + message: RequestResponder[ServerRequest, ClientResult] | ServerNotification | Exception, + ) -> None: + if not isinstance(message, ValueError): + return + if not stream_error.done(): + stream_error.set_result(message) + # The SDK closes pending requests when its message handler raises. + raise RuntimeError("MCP response stream failed") + # Build session kwargs with optional callbacks session_kwargs: Final[dict[str, Any]] = {} if self._sampling_callback is not None: @@ -456,6 +472,7 @@ class MCPClient: read_stream, write_stream, read_timeout_seconds=timedelta(seconds=self.timeout), + message_handler=receive_message if self.transport_type == MCPTransport.http else None, **session_kwargs, ) session: Final = await session_ctx.__aenter__() @@ -467,6 +484,10 @@ class MCPClient: if isinstance(ins, str) and ins.strip(): self._last_initialize_instructions = ins.strip() return await operation(session) + except McpError: + if stream_error.done(): + raise stream_error.result() + raise finally: try: await session_ctx.__aexit__(None, None, None) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 329dddbdf05..c0ffeaeac62 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -3,12 +3,15 @@ import importlib from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime +from traceback import walk_tb from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal +from uuid import uuid4 import anyio import httpx from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from pydantic import ValidationError from starlette.datastructures import Headers from litellm._logging import verbose_logger @@ -30,6 +33,8 @@ from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( list_fault_http_status, outcome_wire_value, ) +from litellm.proxy._experimental.mcp_server.faults.traversal import iter_exception_tree +from litellm.proxy._experimental.mcp_server.oauth_utils import _redact_mcp_resource_url from litellm.proxy._experimental.mcp_server.ui_session_utils import ( acting_user_auth, build_effective_auth_contexts, @@ -78,11 +83,38 @@ _MCP_GUARDRAIL_REJECTIONS: Final = ( def _connection_error_message(exc: BaseException, url: str | None, timeout_seconds: float) -> str: + reference: Final = uuid4().hex + verbose_logger.error( + "MCP connection test failed (reference=%s): %s", + reference, + tuple( + ( + type(cause).__name__, + tuple( + (frame.f_code.co_filename, lineno, frame.f_code.co_name) + for frame, lineno in walk_tb(cause.__traceback__) + ), + ) + for cause in iter_exception_tree(exc) + ), + ) + return next( + ( + message + for cause in iter_exception_tree(exc) + if (message := _known_connection_error_message(cause, url, timeout_seconds)) is not None + ), + "An unexpected error occurred while testing the MCP connection. " + f"Retry; if it persists, share reference {reference} with your gateway administrator.", + ) + + +def _known_connection_error_message(exc: BaseException, url: str | None, timeout_seconds: float) -> str | None: if isinstance(exc, MCPServerURLCredentialsError): return str(exc.detail) if isinstance(exc, TimeoutError): return ( - f"Failed to connect to MCP server: no response from {url or 'the server'} " + f"Failed to connect to MCP server: no response from {_redact_mcp_resource_url(url) or 'the server'} " f"within {timeout_seconds:.0f}s. Check that the LiteLLM proxy can reach this URL " "from its network (DNS, egress rules, firewalls) and that the server answers MCP requests." ) @@ -99,10 +131,32 @@ def _connection_error_message(exc: BaseException, url: str | None, timeout_secon return "Failed to connect to MCP server: the connection timed out." if isinstance(exc, httpx.HTTPStatusError): return f"Failed to connect to MCP server: it returned HTTP {exc.response.status_code}." - return "Failed to connect to MCP server. Check proxy logs for details." + if isinstance(exc, ValueError) and str(exc).startswith("Unexpected content type:"): + return ( + "Failed to connect to MCP server: the endpoint returned an unsupported content type. " + "Check that the URL is an MCP endpoint, not a web page, and matches the selected transport." + ) + if isinstance(exc, ValidationError) and exc.title in ("JSONRPCMessage", "InitializeResult", "ListToolsResult"): + return ( + "Failed to connect to MCP server: the endpoint returned invalid JSON or an invalid MCP response. " + "Check the MCP endpoint URL and the server's protocol implementation." + ) + if MCP_AVAILABLE and isinstance(exc, McpError): + if exc.error.code == 32600 and exc.error.message == "Session terminated": + return ( + "Failed to connect to MCP server: the MCP session was terminated. " + "Check that the URL points to an MCP endpoint and matches the selected transport, " + "then retry to start a new session." + ) + return ( + f"Failed to connect to MCP server: the MCP request failed (JSON-RPC code {exc.error.code}). " + "Check that the endpoint supports MCP initialization and tool listing, and check the upstream server logs." + ) + return None if MCP_AVAILABLE: + from mcp.shared.exceptions import McpError from mcp.types import Tool as MCPTool from litellm.experimental_mcp_client.client import MCPClient @@ -1342,7 +1396,6 @@ if MCP_AVAILABLE: except (KeyboardInterrupt, SystemExit, asyncio.CancelledError): raise except BaseException as e: - verbose_logger.error("Error in MCP operation: %s", e, exc_info=True) return { "status": "error", "error": True, diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index 4db131da62c..49268b4d22d 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -1,9 +1,11 @@ import asyncio import base64 +import json import os import sys from importlib import metadata from pathlib import Path +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import anyio @@ -11,6 +13,8 @@ import httpx import pytest from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth from mcp import McpError +from mcp.client.streamable_http import streamable_http_client +from pydantic import ValidationError from mcp.shared.message import SessionMessage from mcp.types import ( LATEST_PROTOCOL_VERSION, @@ -1224,14 +1228,14 @@ def test_without_a_configured_slot_the_existing_precedence_is_unchanged(): _REDIRECT_CASES = [ - ("https://upstream.example.com/mcp", "https://upstream.example.com/other"), # same origin + ("https://upstream.example.com/mcp", "https://upstream.example.com/other"), # same origin ("https://upstream.example.com/mcp", "https://upstream.example.com:443/other"), # explicit default port - ("https://upstream.example.com/mcp", "https://attacker.example.com/collect"), # different host - ("https://upstream.example.com/mcp", "http://upstream.example.com/collect"), # scheme downgrade - ("https://upstream.example.com/mcp", "https://upstream.example.com:8443/other"), # different port - ("https://upstream.example.com/mcp", "https://sub.upstream.example.com/x"), # different host - ("http://upstream.example.com/mcp", "https://upstream.example.com/other"), # http -> https upgrade - ("http://upstream.example.com/mcp", "http://upstream.example.com/other"), # same origin, plain http + ("https://upstream.example.com/mcp", "https://attacker.example.com/collect"), # different host + ("https://upstream.example.com/mcp", "http://upstream.example.com/collect"), # scheme downgrade + ("https://upstream.example.com/mcp", "https://upstream.example.com:8443/other"), # different port + ("https://upstream.example.com/mcp", "https://sub.upstream.example.com/x"), # different host + ("http://upstream.example.com/mcp", "https://upstream.example.com/other"), # http -> https upgrade + ("http://upstream.example.com/mcp", "http://upstream.example.com/other"), # same origin, plain http ] @@ -1283,3 +1287,109 @@ def test_a_differently_cased_injected_header_cannot_shadow_the_slot() -> None: headers = client._get_auth_headers() assert [v for k, v in headers.items() if k.lower() == "esb-oauth"] == ["Bearer minted-token"] assert headers["X-Trace"] == "keep" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("content_type", "body", "expected_type"), + [ + ("text/html", b"secret-page", ValueError), + ("application/json", b"secret-invalid-json", ValidationError), + ("application/json", b'{"secret":"invalid-rpc"}', ValidationError), + ("application/json", b'{"jsonrpc":"2.0","id":0,"result":{"secret":"invalid-schema"}}', ValidationError), + ], +) +async def test_invalid_http_response_surfaces_without_waiting_for_timeout( + content_type: str, body: bytes, expected_type: type[Exception] +) -> None: + from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message + + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"Content-Type": content_type}, content=body) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) + with pytest.raises(expected_type) as caught: + await asyncio.wait_for( + client._execute_session_operation( + streamable_http_client(client.server_url, http_client=http_client), + lambda session: session.list_tools(), + ), + timeout=3, + ) + + message: Final = _connection_error_message(caught.value, client.server_url, 30) + assert "unsupported content type" in message or "invalid MCP response" in message + assert "secret" not in message + assert "timed out" not in message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", [200, 401, 503]) +async def test_http_response_handler_preserves_success_and_http_errors(status_code: int) -> None: + def respond(request: httpx.Request) -> httpx.Response: + if request.method == "DELETE": + return httpx.Response(200) + payload: Final = json.loads(request.content) + if "id" not in payload: + return httpx.Response(202) + result: Final = ( + { + "protocolVersion": LATEST_PROTOCOL_VERSION, + "capabilities": {}, + "serverInfo": {"name": "test", "version": "1"}, + } + if payload["method"] == "initialize" + else {"tools": []} + ) + return httpx.Response(status_code, json={"jsonrpc": "2.0", "id": payload["id"], "result": result}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) + operation: Final = client._execute_session_operation( + streamable_http_client(client.server_url, http_client=http_client), lambda session: session.list_tools() + ) + if status_code == 200: + result: Final = await asyncio.wait_for(operation, timeout=3) + assert result.tools == [] + else: + with pytest.raises(httpx.HTTPStatusError) as caught: + await asyncio.wait_for(operation, timeout=3) + assert caught.value.response.status_code == status_code + + +@pytest.mark.asyncio +async def test_invalid_tool_list_schema_is_identified_as_an_upstream_response() -> None: + from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message + + def respond(request: httpx.Request) -> httpx.Response: + if request.method == "DELETE": + return httpx.Response(200) + payload: Final = json.loads(request.content) + if "id" not in payload: + return httpx.Response(202) + result: Final = ( + { + "protocolVersion": LATEST_PROTOCOL_VERSION, + "capabilities": {}, + "serverInfo": {"name": "test", "version": "1"}, + } + if payload["method"] == "initialize" + else {"tools": "secret-invalid-tools"} + ) + return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload["id"], "result": result}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) + with pytest.raises(ValidationError) as caught: + await asyncio.wait_for( + client._execute_session_operation( + streamable_http_client(client.server_url, http_client=http_client), + lambda session: session.list_tools(), + ), + timeout=3, + ) + + message: Final = _connection_error_message(caught.value, client.server_url, 30) + assert "invalid MCP response" in message + assert "secret" not in message diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index bd692776c82..73734131130 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -3,7 +3,7 @@ import inspect import json import sys from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11 @@ -113,7 +113,7 @@ class TestExecuteWithMcpClient: assert "stack_trace" not in result @pytest.mark.asyncio - async def test_timeout_caps_hanging_operation_and_names_url(self, monkeypatch): + async def test_timeout_caps_hanging_operation_and_names_origin(self, monkeypatch): async def fake_create_client(*args, **kwargs): return object() @@ -138,7 +138,7 @@ class TestExecuteWithMcpClient: ) assert result["error"] is True - assert "https://mcp.example.com/mcp/" in result["message"] + assert "https://mcp.example.com" in result["message"] @pytest.mark.asyncio async def test_timeout_covers_client_creation(self, monkeypatch): @@ -166,15 +166,15 @@ class TestExecuteWithMcpClient: ) assert result["error"] is True - assert "https://mcp.example.com/mcp/" in result["message"] + assert "https://mcp.example.com" in result["message"] def test_timeout_defaults_to_tool_listing_timeout(self): default = inspect.signature(rest_endpoints._execute_with_mcp_client).parameters["timeout_seconds"].default assert default == MCP_TOOL_LISTING_TIMEOUT - def test_connection_error_message_timeout_names_url_and_budget(self): + def test_connection_error_message_timeout_names_origin_and_budget(self): message = rest_endpoints._connection_error_message(TimeoutError(), "https://api.example.com/mcp/", 30.0) - assert "https://api.example.com/mcp/" in message + assert "https://api.example.com" in message assert "30s" in message def test_connection_error_message_hides_arbitrary_http_exception_detail(self): @@ -592,7 +592,7 @@ class TestExecuteWithMcpClient: assert result["status"] == "error" assert result["error"] is True - assert "Failed to connect to MCP server" in result["message"] + assert "reference" in result["message"] # Error message must not leak raw exception details assert "cancel scope" not in result["message"] @@ -3430,7 +3430,139 @@ class TestConnectionErrorMessage: def test_unknown_error_falls_back_to_generic(self): message = rest_endpoints._connection_error_message(RuntimeError("weird"), "https://example.com", 30.0) assert "weird" not in message - assert "proxy logs" in message.lower() + assert "reference" in message.lower() + + def test_sdk_session_terminated_explains_endpoint_and_retry(self) -> None: + from mcp.shared.exceptions import McpError + from mcp.types import ErrorData + + message: Final = rest_endpoints._connection_error_message( + McpError(ErrorData(code=32600, message="Session terminated")), "https://example.com/mcp", 30.0 + ) + + assert "session was terminated" in message + assert "MCP endpoint" in message + assert "transport" in message + assert "retry" in message + assert "404" not in message + + @pytest.mark.parametrize("code", [-32700, -32601, -32602, -32603, -32000, 32600, 408]) + def test_rpc_errors_include_code_without_echoing_upstream_data(self, code: int) -> None: + from mcp.shared.exceptions import McpError + from mcp.types import ErrorData + + message: Final = rest_endpoints._connection_error_message( + McpError(ErrorData(code=code, message="secret-message", data={"token": "secret-data"})), + "https://example.com/secret-path?token=secret-query", + 30.0, + ) + + assert f"JSON-RPC code {code}" in message + assert "secret" not in message + assert "timed out" not in message + assert "session was terminated" not in message + + @pytest.mark.parametrize("status_code", [401, 403, 404, 405, 429, 503]) + def test_wrapped_http_failures_preserve_status(self, status_code: int) -> None: + response: Final = httpx.Response(status_code, text="secret-body") + upstream: Final = httpx.HTTPStatusError( + "secret-exception", + request=httpx.Request("POST", "https://example.com/?token=secret-query"), + response=response, + ) + wrapped: Final = BaseExceptionGroup( + "secret-group", [asyncio.CancelledError(), BaseExceptionGroup("nested", [upstream])] + ) + + message: Final = rest_endpoints._connection_error_message(wrapped, "https://example.com", 30.0) + + assert f"HTTP {status_code}" in message + assert "secret" not in message + + def test_explicit_cause_is_classified_before_incidental_context(self) -> None: + wrapped: Final = RuntimeError("secret-wrapper") + wrapped.__cause__ = httpx.ConnectError("secret-cause") + wrapped.__context__ = TimeoutError("secret-context") + + message: Final = rest_endpoints._connection_error_message(wrapped, "https://example.com", 30.0) + + assert "unreachable" in message + assert "secret" not in message + + def test_timeout_url_redacts_credentials_path_query_and_fragment(self) -> None: + message: Final = rest_endpoints._connection_error_message( + TimeoutError("secret-error"), + "https://secret-user:secret-pass@example.com:8443/secret-path?token=secret-query#secret-fragment", + 30.0, + ) + + assert "https://example.com:8443" in message + assert "30s" in message + assert "secret" not in message + + def test_unknown_failure_reference_matches_safe_diagnostics(self, caplog: pytest.LogCaptureFixture) -> None: + import re + + try: + raise RuntimeError("secret-exception-body") + except RuntimeError as exc: + message: Final = rest_endpoints._connection_error_message( + exc, "https://secret-user:secret-password@example.com/secret-path?token=secret-query", 30.0 + ) + + reference: Final = re.search(r"reference ([a-f0-9]{32})", message) + assert reference is not None + diagnostics: Final = tuple( + record for record in caplog.records if "MCP connection test failed" in record.message + ) + assert len(diagnostics) == 1 + assert reference.group(1) in diagnostics[0].message + assert "RuntimeError" in diagnostics[0].message + assert "test_unknown_failure_reference_matches_safe_diagnostics" in diagnostics[0].message + assert diagnostics[0].exc_info is None + assert "secret" not in message + diagnostics[0].message + + @pytest.mark.parametrize("exc", [ValueError("secret-config"), HTTPException(500, "secret-detail")]) + def test_unrelated_errors_are_not_misreported_as_invalid_mcp(self, exc: Exception) -> None: + message: Final = rest_endpoints._connection_error_message(exc, "https://example.com", 30.0) + + assert "reference" in message + assert "invalid MCP response" not in message + assert "secret" not in message + + def test_configuration_validation_error_uses_unknown_fallback(self) -> None: + from pydantic import ValidationError + + with pytest.raises(ValidationError) as caught: + NewMCPServerRequest.model_validate({"server_name": "example", "transport": "secret-invalid-transport"}) + + message: Final = rest_endpoints._connection_error_message(caught.value, "https://example.com", 30.0) + assert "reference" in message + assert "invalid MCP response" not in message + assert "secret" not in message + + @pytest.mark.asyncio + async def test_connection_test_preserves_cancellation(self) -> None: + async def cancelled_operation(client: rest_endpoints.MCPClient) -> dict[str, object]: + raise asyncio.CancelledError + + payload: Final = NewMCPServerRequest(server_name="cancelled", url="https://example.com", auth_type=MCPAuth.none) + with pytest.raises(asyncio.CancelledError): + await rest_endpoints._execute_with_mcp_client(payload, cancelled_operation) + + @pytest.mark.asyncio + async def test_unknown_failure_preserves_response_contract(self) -> None: + async def failing_operation(client: rest_endpoints.MCPClient) -> dict[str, object]: + raise RuntimeError("secret-operation") + + payload: Final = NewMCPServerRequest(server_name="unknown", url="https://example.com", auth_type=MCPAuth.none) + result: Final = await rest_endpoints._execute_with_mcp_client(payload, failing_operation) + + assert result["error"] is True + assert result["status"] == "error" + assert "reference" in result["message"] + assert "secret" not in result["message"] + assert "stack_trace" not in result class TestGetServerAuthHeaderGroupDefault: From 7b6ef9206e87c09adfc5844c8f8130da73d794fb Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:14:47 -0700 Subject: [PATCH 172/411] test(mcp): cover server notifications during tool listing --- .../test_mcp_client.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index 49268b4d22d..90292b9b162 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -24,6 +24,7 @@ from mcp.types import ( JSONRPCError, JSONRPCMessage, JSONRPCResponse, + LoggingMessageNotificationParams, ServerCapabilities, ) @@ -1358,6 +1359,58 @@ async def test_http_response_handler_preserves_success_and_http_errors(status_co assert caught.value.response.status_code == status_code +@pytest.mark.asyncio +async def test_http_response_handler_preserves_notifications_and_tool_listing() -> None: + notification: Final = { + "jsonrpc": "2.0", + "method": "notifications/message", + "params": {"level": "info", "data": "Listing tools"}, + } + logging_callback: Final = AsyncMock() + + def respond(request: httpx.Request) -> httpx.Response: + if request.method == "DELETE": + return httpx.Response(200) + payload: Final = json.loads(request.content) + if "id" not in payload: + return httpx.Response(202) + if payload["method"] == "initialize": + return httpx.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload["id"], + "result": { + "protocolVersion": LATEST_PROTOCOL_VERSION, + "capabilities": {"logging": {}, "tools": {}}, + "serverInfo": {"name": "test", "version": "1"}, + }, + }, + ) + response: Final = { + "jsonrpc": "2.0", + "id": payload["id"], + "result": {"tools": [{"name": "search", "inputSchema": {"type": "object"}}]}, + } + return httpx.Response( + 200, + headers={"Content-Type": "text/event-stream"}, + content="".join(f"event: message\ndata: {json.dumps(message)}\n\n" for message in (notification, response)), + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30, logging_callback=logging_callback) + result: Final = await asyncio.wait_for( + client._execute_session_operation( + streamable_http_client(client.server_url, http_client=http_client), lambda session: session.list_tools() + ), + timeout=3, + ) + + assert [tool.name for tool in result.tools] == ["search"] + logging_callback.assert_awaited_once_with(LoggingMessageNotificationParams(level="info", data="Listing tools")) + + @pytest.mark.asyncio async def test_invalid_tool_list_schema_is_identified_as_an_upstream_response() -> None: from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message From 884f90c72715b0cfe0d5e2ea09f3744cb8376a78 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 05:15:53 +0000 Subject: [PATCH 173/411] refactor(cost-map): inline background retry, trim tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/get_model_cost_map.py | 119 ++++----- .../test_get_model_cost_map.py | 243 +++++++----------- 2 files changed, 139 insertions(+), 223 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 67d06a0758e..f81ddbfee2e 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -184,10 +184,6 @@ def mark_litellm_import_complete() -> None: _litellm_import_complete.set() -def _start_daemon_thread(fn: Callable[[], None]) -> None: - threading.Thread(target=fn, name="litellm-model-cost-map-retry", daemon=True).start() - - @dataclass(frozen=True, slots=True) class ModelCostMapReloaded: model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict @@ -531,32 +527,6 @@ def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMa return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) -def _use_local_backup(reason: str | None) -> dict: # mutable-ok: returns the mutable model-cost map contract - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = reason - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map - - -def _accept_remote( - result: ModelCostMapReloaded, url: str -) -> dict | None: # mutable-ok: returns the mutable model-cost map contract - if not GetModelCostMap.validate_model_cost_map( - fetched_map=result.model_cost_map, - backup_model_count=GetModelCostMap._get_backup_model_count(), # pyright: ignore[reportPrivateUsage] # integrity cache - ): - verbose_logger.warning( - "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", - url, - ) - return None - finalized: Final = _finalize_loaded_model_cost_map(result).model_cost_map - _cost_map_source_info.source = "remote" - _cost_map_source_info.url = url - _cost_map_source_info.is_env_forced = False - _cost_map_source_info.fallback_reason = None - return finalized - - def adopt_model_cost_map( new_model_cost_map: dict, # mutable-ok: public API preserves the mutable cost-map contract ) -> int: @@ -579,7 +549,6 @@ def _retry_remote_fetch_in_background( rng: random.Random, client: _SyncGetClient, first_outcome: _FetchAttemptRetryable, - apply: Callable[[dict], object], # mutable-ok: injected callback receives the mutable cost-map dict ) -> None: try: first_wait: Final = _next_retry_wait(outcome=first_outcome, attempt=1, max_attempts=max_attempts, rng=rng) @@ -602,12 +571,20 @@ def _retry_remote_fetch_in_background( ) return _litellm_import_complete.wait() - accepted: Final = _accept_remote(result, url) - if accepted is None: - _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" + if not GetModelCostMap.validate_model_cost_map( + fetched_map=result.model_cost_map, + backup_model_count=GetModelCostMap._get_backup_model_count(), # pyright: ignore[reportPrivateUsage] # integrity cache + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", + url, + ) return - apply(accepted) + finalized: Final = _finalize_loaded_model_cost_map(result).model_cost_map + _cost_map_source_info.source = "remote" + _cost_map_source_info.fallback_reason = None _cost_map_source_info.loaded_at = datetime.now(timezone.utc) + adopt_model_cost_map(finalized) except Exception as e: verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e) @@ -619,19 +596,12 @@ def get_model_cost_map( sleep: Callable[[float], None] = time.sleep, rng: random.Random | None = None, client: "_SyncGetClient | None" = None, - start_background: Callable[[Callable[[], None]], None] = _start_daemon_thread, - apply: Callable[ # mutable-ok: injected callback receives the mutable cost-map dict - [dict], - object, - ] = adopt_model_cost_map, ) -> dict: """ Public entry point — returns the model cost map dict. 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only. - 2. Otherwise fetches from ``url``, validates the first response, and falls - back to the local backup while retrying transient HTTP errors in the - background. + 2. Otherwise fetches from ``url``, retrying transient errors in a background thread. Only the backup model count is cached (a single int) for validation. The full backup dict is only parsed when it must be *returned* as a @@ -641,9 +611,11 @@ def get_model_cost_map( # Note: can't use get_secret_bool here — this runs during litellm.__init__ # before litellm._key_management_settings is set. if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + _cost_map_source_info.source = "local" _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True - return _use_local_backup(None) + _cost_map_source_info.fallback_reason = None + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False @@ -651,32 +623,45 @@ def get_model_cost_map( fetch_client: Final = client if client is not None else httpx fetch_rng: Final = rng if rng is not None else random.Random() outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout) - if isinstance(outcome, ModelCostMapReloaded): - accepted: Final = _accept_remote(outcome, url) - return accepted if accepted is not None else _use_local_backup("Remote data failed integrity validation") if isinstance(outcome, _FetchAttemptRetryable) and max_attempts > 1: - verbose_logger.warning( - "LiteLLM: model cost map fetch attempt 1/%d failed: %s; " - "using local backup while retrying in the background", - max_attempts, - outcome.reason, - ) - start_background( - lambda: _retry_remote_fetch_in_background( - url=url, - timeout=timeout, - max_attempts=max_attempts, - sleep=sleep, - rng=fetch_rng, - client=fetch_client, - first_outcome=outcome, - apply=apply, - ) - ) - else: + threading.Thread( + target=_retry_remote_fetch_in_background, + kwargs={ # mutable-ok: threading requires a mutable keyword-arguments mapping + "url": url, + "timeout": timeout, + "max_attempts": max_attempts, + "sleep": sleep, + "rng": fetch_rng, + "client": fetch_client, + "first_outcome": outcome, + }, + name="litellm-model-cost-map-retry", + daemon=True, + ).start() + if not isinstance(outcome, ModelCostMapReloaded): verbose_logger.warning( "LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.", url, outcome.reason, ) - return _use_local_backup(f"Remote fetch failed: {outcome.reason}") + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = f"Remote fetch failed: {outcome.reason}" + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + content: Final = outcome.model_cost_map + + # Validate using cached count (cheap int comparison, no file I/O) + if not GetModelCostMap.validate_model_cost_map( + fetched_map=content, + backup_model_count=GetModelCostMap._get_backup_model_count(), + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", + url, + ) + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + + _cost_map_source_info.source = "remote" + _cost_map_source_info.fallback_reason = None + return _finalize_loaded_model_cost_map(outcome).model_cost_map diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index e39658a8f37..266c2ca1465 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -6,6 +6,7 @@ count actual model entries, not reserved meta keys) and the extraction of the import json import os +import threading import pytest @@ -20,7 +21,6 @@ from litellm.litellm_core_utils.get_model_cost_map import ( GetModelCostMap, _count_model_entries, _finalize_model_cost_map, - adopt_model_cost_map, get_model_cost_map_provenance, git_blob_id, ) @@ -566,194 +566,125 @@ from litellm.litellm_core_utils.get_model_cost_map import ( class _SyncSleepRecorder: """Injected in place of time.sleep so the boot path's waits are asserted without delay.""" - def __init__(self): + def __init__(self, block=False): self.waits = [] + self.block = block + self.started = threading.Event() + self.release = threading.Event() def __call__(self, seconds: float) -> None: + if self.block: + self.started.set() + self.release.wait(timeout=10) self.waits.append(seconds) -class _BackgroundRecorder: - def __init__(self): - self.callbacks = [] - - def __call__(self, callback): - self.callbacks.append(callback) +def _retry_threads(): + return [thread for thread in threading.enumerate() if thread.name == "litellm-model-cost-map-retry"] -def test_boot_load_returns_local_map_and_schedules_transient_retry(): - client, calls = _mock_client( - [ - httpx.ConnectError("connection refused"), - ], - client_cls=httpx.Client, - ) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() - - cost_map = get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - start_background=background, - ) - assert calls["count"] == 1 - assert sleeper.waits == [] - assert len(background.callbacks) == 1 - assert len(cost_map) > 100 - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert source["fallback_reason"] is not None - - -def test_background_retry_adopts_valid_remote_map(): - client, calls = _mock_client( - [ - httpx.ConnectError("connection refused"), - httpx.Response(200, content=_real_map_bytes()), - ], - client_cls=httpx.Client, - ) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() - applied = [] - - get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - start_background=background, - apply=applied.append, - ) - assert calls["count"] == 1 - assert sleeper.waits == [] - assert len(background.callbacks) == 1 - - background.callbacks[0]() - - assert calls["count"] == 2 - assert len(sleeper.waits) == 1 - assert 2.0 <= sleeper.waits[0] < 3.0 - assert len(applied) == 1 - assert applied[0].keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} - source = get_model_cost_map_source_info() - assert source["source"] == "remote" - assert source["fallback_reason"] is None - - -def test_background_retry_keeps_local_map_after_remaining_failures(): - client, calls = _mock_client( - [httpx.ConnectError("connection refused"), httpx.Response(503)], - client_cls=httpx.Client, - ) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() - applied = [] - - get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - start_background=background, - apply=applied.append, - ) - background.callbacks[0]() - - assert calls["count"] == 3 - assert len(sleeper.waits) == 2 - assert 2.0 <= sleeper.waits[0] < 3.0 - assert 4.0 <= sleeper.waits[1] < 5.0 - assert applied == [] - assert get_model_cost_map_source_info()["source"] == "local" - - -def test_boot_load_does_not_schedule_non_retryable_failure(): - client, calls = _mock_client([httpx.Response(404)], client_cls=httpx.Client) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() - - get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - start_background=background, - ) - - assert calls["count"] == 1 - assert sleeper.waits == [] - assert background.callbacks == [] - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert source["fallback_reason"] is not None - - -def test_boot_load_success_does_not_schedule_background_retry(): +def test_boot_load_success_does_not_start_background_retry(): client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() cost_map = get_model_cost_map( url=_URL, sleep=sleeper, rng=random.Random(0), client=client, - start_background=background, ) assert calls["count"] == 1 assert sleeper.waits == [] - assert background.callbacks == [] - assert get_model_cost_map_source_info()["source"] == "remote" + assert _retry_threads() == [] assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} + assert get_model_cost_map_source_info()["source"] == "remote" -def test_boot_load_with_one_attempt_does_not_schedule_background_retry(): - client, calls = _mock_client([httpx.ConnectError("connection refused")], client_cls=httpx.Client) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() +def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_remote(monkeypatch): + import litellm + from litellm import utils as litellm_utils + from litellm.litellm_core_utils import get_model_cost_map as module - get_model_cost_map( + original_model_cost = litellm.model_cost + monkeypatch.setattr(litellm, "model_cost", dict(original_model_cost)) + for name, provider_models in tuple(vars(litellm).items()): + if name.endswith("_models") and isinstance(provider_models, set): + monkeypatch.setattr(litellm, name, set(provider_models)) + monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider)) + monkeypatch.setattr( + litellm_utils, + "_runtime_registered_model_cost", + dict(litellm_utils._runtime_registered_model_cost), + ) + source_info = module._cost_map_source_info + for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"): + monkeypatch.setattr(source_info, name, getattr(source_info, name)) + + remote_map = _load_root_cost_map() + remote_map["claude-remote-only-test"] = {"litellm_provider": "anthropic", "mode": "chat"} + client, calls = _mock_client( + [ + httpx.ConnectError("connection refused"), + httpx.Response(200, content=json.dumps(remote_map).encode()), + ], + client_cls=httpx.Client, + ) + sleeper = _SyncSleepRecorder(block=True) + litellm.register_model({"my-runtime-model": {"litellm_provider": "custom", "max_input_tokens": 4321}}) + + cost_map = get_model_cost_map( url=_URL, - max_attempts=1, + max_attempts=3, sleep=sleeper, rng=random.Random(0), client=client, - start_background=background, ) assert calls["count"] == 1 assert sleeper.waits == [] - assert background.callbacks == [] - assert get_model_cost_map_source_info()["source"] == "local" - - -def test_adopt_model_cost_map_replays_runtime_registration_and_provider_models(): - import litellm - from litellm import utils as litellm_utils - - original_model_cost = litellm.model_cost - original_registry = dict(litellm_utils._runtime_registered_model_cost) - original_anthropic_models = set(litellm.anthropic_models) + assert sleeper.started.wait(timeout=10) + threads = _retry_threads() try: - litellm.register_model( - model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}} - ) - - models_count = adopt_model_cost_map({"anthropic/new-model": {"litellm_provider": "anthropic", "mode": "chat"}}) - - assert models_count == 1 - assert "anthropic/new-model" in litellm.anthropic_models - assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321 + assert len(threads) == 1 + assert "claude-remote-only-test" not in cost_map + assert cost_map.keys() == _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()).keys() + source = get_model_cost_map_source_info() + assert source["source"] == "local" + assert source["fallback_reason"].startswith("Remote fetch failed:") + sleeper.release.set() + for thread in threads: + thread.join(timeout=10) + assert all(not thread.is_alive() for thread in threads) + assert sleeper.waits and 2.0 <= sleeper.waits[0] < 3.0 + assert calls["count"] == 2 + assert "claude-remote-only-test" in litellm.model_cost + assert "claude-remote-only-test" in litellm.anthropic_models + assert "my-runtime-model" in litellm.model_cost + source = get_model_cost_map_source_info() + assert source["source"] == "remote" + assert source["fallback_reason"] is None finally: - litellm.model_cost = original_model_cost # test-quality-ok: restore the module state changed by adoption - litellm_utils._runtime_registered_model_cost.clear() - litellm_utils._runtime_registered_model_cost.update(original_registry) - litellm.anthropic_models.clear() - litellm.anthropic_models.update(original_anthropic_models) - litellm_utils._invalidate_model_cost_lowercase_map() + sleeper.release.set() + for thread in _retry_threads(): + thread.join(timeout=10) + + +def test_boot_load_does_not_retry_non_retryable_failure(): + client, calls = _mock_client([httpx.Response(404)], client_cls=httpx.Client) + sleeper = _SyncSleepRecorder() + + get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + ) + assert calls["count"] == 1 + assert sleeper.waits == [] + assert _retry_threads() == [] + source = get_model_cost_map_source_info() + assert source["source"] == "local" + assert source["fallback_reason"] is not None def test_boot_load_respects_local_env_override(monkeypatch): From 0721163cacfbd9fbfeee5dd25205cb76840111dd Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 8 Sep 2026 22:49:40 -0700 Subject: [PATCH 174/411] test(e2e/ui): cover team-scoped model visibility, re-editing litellm params, and model health checks (#40039) * test(e2e/ui): cover team-scoped model visibility, re-editing litellm params, and model health checks Three Models and Endpoints flows had no end-to-end coverage, and all three keep coming back as bug reports. modelsByTeam walks an internal user through the Current team control and asserts the table lists exactly what each team grants. It creates one deployment that belongs to no team, proves that deployment is visible under Personal, then proves it is absent under both seeded teams, so an empty table cannot pass the same assertions. editLitellmParams adds a temperature and a custom pair to a deployment, saves, then re-edits the temperature and drops the custom pair. It checks both update request bodies, polls the stored deployment until the new temperature is there, reloads the page to confirm the second save is what renders, and sends one chat completion to prove the deployment still serves. modelHealthStatus runs the health check on a reachable deployment and on one pointed at a dead port, asserts the healthy and unhealthy cells and the two detail dialogs, and reloads to confirm both statuses are stored. Every deployment these specs create carries a unique name and is deleted in afterEach, including on the failure path. * test(e2e/ui): find health rows across every page of the health table The health table pages server-side at 50 rows with no search box, so on a proxy carrying more deployments than that the two deployments the spec creates can land on a later page and the lookup finds nothing. Row lookups now walk the pages, using the table's own page indicator to know when to advance and when to wrap back to the first page. * test(e2e/ui): build the created deployment ids without mutating the array * test(ui): scope model deployments to Playwright fixtures --- .../tests/internal-user/modelsByTeam.spec.ts | 196 ++++++++++++++ .../modelsPage/editLitellmParams.spec.ts | 252 ++++++++++++++++++ .../modelsPage/modelHealthStatus.spec.ts | 245 +++++++++++++++++ 3 files changed, 693 insertions(+) create mode 100644 tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts create mode 100644 tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts create mode 100644 tests/e2e/ui/tests/modelsPage/modelHealthStatus.spec.ts diff --git a/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts b/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts new file mode 100644 index 00000000000..5e2c80b5845 --- /dev/null +++ b/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts @@ -0,0 +1,196 @@ +import { + test as base, + expect, + type Locator, + type Page as PlaywrightPage, +} from "@playwright/test"; +import { + E2E_TEAM_CRUD_ALIAS, + E2E_TEAM_ORG_ALIAS, + INTERNAL_USER_STORAGE_PATH, +} from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; +import { readBack } from "../../helpers/roundTrip"; +import { CHAT_MODEL_A, CHAT_MODEL_B, masterKey } from "../../helpers/traffic"; + +const MOCK_LLM_BASE = `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`; +const CURRENT_TEAM_VIEW = "Current Team Models"; +const ALL_MODELS_VIEW = "All Available Models"; +const PERSONAL_TEAM = "Personal"; + +const teamSelector = (page: PlaywrightPage): Locator => + page.getByRole("combobox", { name: "Current team", exact: true }); +const viewSelector = (page: PlaywrightPage): Locator => + page.getByRole("combobox", { name: "View", exact: true }); + +async function chooseOption( + page: PlaywrightPage, + selector: Locator, + optionName: string, +): Promise { + await selector.click(); + const option = page.getByRole("option", { name: optionName, exact: true }); + await expect(option, `option ${optionName} is offered`).toBeVisible({ + timeout: 10_000, + }); + await option.click(); + await expect( + selector, + `${optionName} is the selection the control now reports`, + ).toContainText(optionName, { + timeout: 10_000, + }); +} + +async function deleteDeployment( + page: PlaywrightPage, + id: string, +): Promise { + const post = () => + page.request.post("/model/delete", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { id }, + }); + const deleted = await post().catch(() => post()); + expect( + deleted.ok(), + `cleanup: /model/delete ${id} returned ${deleted.status()}`, + ).toBe(true); +} + +function modelRow(page: PlaywrightPage, modelName: string): Locator { + return page.getByRole("row").filter({ hasText: modelName }); +} + +async function isRegistered( + page: PlaywrightPage, + modelName: string, +): Promise { + const body = await readBack<{ data: { model_name?: string }[] }>( + page, + "/v2/model/info", + ); + return body.data.some((row) => row.model_name === modelName); +} + +const uniqueSuffix = (): string => + `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + +const test = base.extend<{ ungrantedModelName: string }>({ + ungrantedModelName: async ({ page }, use) => { + const ungrantedModelName = `e2e-ungranted-${uniqueSuffix()}`; + const created = await page.request.post("/model/new", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { + model_name: ungrantedModelName, + litellm_params: { + model: `openai/${ungrantedModelName}`, + api_base: MOCK_LLM_BASE, + api_key: "fake-key", + }, + model_info: {}, + }, + }); + expect( + created.ok(), + `/model/new failed: ${created.status()} ${await created.text()}`, + ).toBe(true); + const ungrantedModelId = (await created.json()).model_info?.id; + expect(ungrantedModelId, "model id from /model/new").toBeTruthy(); + + try { + await expect + .poll(async () => await isRegistered(page, ungrantedModelName), { + message: `deployment ${ungrantedModelName} never appeared in /v2/model/info after create`, + timeout: 60_000, + }) + .toBe(true); + await use(ungrantedModelName); + } finally { + await deleteDeployment(page, ungrantedModelId); + } + }, +}); + +test.describe("Models and Endpoints for an internal user", () => { + test.use({ storageState: INTERNAL_USER_STORAGE_PATH }); + + test("shows an internal user exactly the models of the team they select", async ({ + page, + ungrantedModelName, + }) => { + await navigateToPage(page, Page.Models); + + await expect( + page.getByRole("tab", { name: "Your Models" }), + "an internal user lands on their own models tab, not an admin-only view", + ).toBeVisible({ timeout: 15_000 }); + await expect( + viewSelector(page), + "the models table opens scoped to the selected team", + ).toContainText(CURRENT_TEAM_VIEW, { timeout: 15_000 }); + await expect( + modelRow(page, ungrantedModelName), + `the personal view lists ${ungrantedModelName}, so it is on the proxy and reachable from this page`, + ).toHaveCount(1, { timeout: 30_000 }); + + await chooseOption(page, teamSelector(page), E2E_TEAM_CRUD_ALIAS); + await expect( + modelRow(page, CHAT_MODEL_A), + `${E2E_TEAM_CRUD_ALIAS} lists ${CHAT_MODEL_A}`, + ).toHaveCount(1, { + timeout: 15_000, + }); + await expect( + modelRow(page, CHAT_MODEL_B), + `${E2E_TEAM_CRUD_ALIAS} lists ${CHAT_MODEL_B}`, + ).toHaveCount(1, { + timeout: 15_000, + }); + await expect( + modelRow(page, ungrantedModelName), + `${ungrantedModelName} is on the proxy but not granted to ${E2E_TEAM_CRUD_ALIAS}, so it must not be listed`, + ).toHaveCount(0); + + await chooseOption(page, teamSelector(page), E2E_TEAM_ORG_ALIAS); + await expect( + modelRow(page, CHAT_MODEL_A), + `${E2E_TEAM_ORG_ALIAS} lists ${CHAT_MODEL_A}`, + ).toHaveCount(1, { + timeout: 15_000, + }); + await expect( + page.getByTestId("pagination-range"), + `${E2E_TEAM_ORG_ALIAS} lists the one model it grants and nothing else`, + ).toHaveText("Showing 1-1 of 1", { timeout: 15_000 }); + await expect( + modelRow(page, CHAT_MODEL_B), + `${CHAT_MODEL_B} belongs to another team and must not leak into ${E2E_TEAM_ORG_ALIAS}`, + ).toHaveCount(0); + await expect( + modelRow(page, ungrantedModelName), + `${ungrantedModelName} is granted to no team and must not leak into ${E2E_TEAM_ORG_ALIAS}`, + ).toHaveCount(0); + + await chooseOption(page, viewSelector(page), ALL_MODELS_VIEW); + await expect( + modelRow(page, CHAT_MODEL_A), + `switching to ${ALL_MODELS_VIEW} leaves the table populated rather than blanking it`, + ).toHaveCount(1, { timeout: 15_000 }); + + await page.reload(); + await expect( + teamSelector(page), + "the team selection is not persisted across a reload, so the table returns to the personal view", + ).toContainText(PERSONAL_TEAM, { timeout: 15_000 }); + await expect( + viewSelector(page), + "the view selection is not persisted across a reload either", + ).toContainText(CURRENT_TEAM_VIEW, { timeout: 15_000 }); + await expect( + modelRow(page, ungrantedModelName), + "the personal view still renders models after a reload rather than coming back empty", + ).toHaveCount(1, { timeout: 30_000 }); + }); +}); diff --git a/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts b/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts new file mode 100644 index 00000000000..4480515ae59 --- /dev/null +++ b/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts @@ -0,0 +1,252 @@ +import { + test as base, + expect, + type Page as PlaywrightPage, +} from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; +import { captureRequestBody, readBack } from "../../helpers/roundTrip"; +import { masterKey, sendChatCompletion } from "../../helpers/traffic"; + +const MOCK_LLM_BASE = `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`; +const CUSTOM_PARAM = "extra_headers"; +const CUSTOM_PARAM_VALUE = { "X-E2E-Edit-Probe": "one" }; + +type StoredParams = Record; + +async function readStoredParams( + page: PlaywrightPage, + modelId: string, +): Promise { + const body = await readBack<{ data: { litellm_params: StoredParams }[] }>( + page, + `/model/info?litellm_model_id=${modelId}`, + ); + return body.data[0]?.litellm_params ?? {}; +} + +function paramsEditor(page: PlaywrightPage) { + return page.getByPlaceholder('"rpm": 100'); +} + +async function editParams( + page: PlaywrightPage, + mutate: (params: StoredParams) => StoredParams, +): Promise { + await page.getByRole("button", { name: "Edit Settings" }).click(); + const editor = paramsEditor(page); + await expect( + editor, + "the LiteLLM Params editor is reachable on every visit to the edit form", + ).toBeVisible({ + timeout: 15_000, + }); + const shown = JSON.parse(await editor.inputValue()) as StoredParams; + await editor.fill(JSON.stringify(mutate(shown), null, 2)); +} + +async function deleteDeployment( + page: PlaywrightPage, + id: string, +): Promise { + const post = () => + page.request.post("/model/delete", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { id }, + }); + const deleted = await post().catch(() => post()); + expect( + deleted.ok(), + `cleanup: /model/delete ${id} returned ${deleted.status()}`, + ).toBe(true); +} + +const uniqueSuffix = (): string => + `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + +const test = base.extend<{ + deployment: { readonly modelName: string; readonly createdModelId: string }; +}>({ + deployment: async ({ page, request }, use) => { + const modelName = `e2e-edit-params-${uniqueSuffix()}`; + const created = await page.request.post("/model/new", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { + model_name: modelName, + litellm_params: { + model: `openai/${modelName}`, + api_base: MOCK_LLM_BASE, + api_key: "fake-key", + }, + model_info: {}, + }, + }); + expect( + created.ok(), + `/model/new failed: ${created.status()} ${await created.text()}`, + ).toBe(true); + const createdModelId = (await created.json()).model_info?.id; + expect(createdModelId, "model id from /model/new").toBeTruthy(); + + try { + await expect + .poll( + async () => { + try { + await sendChatCompletion(request, { + model: modelName, + prompt: `warmup ${modelName}`, + }); + return true; + } catch { + return false; + } + }, + { + message: `deployment ${modelName} never became routable after /model/new`, + timeout: 60_000, + }, + ) + .toBe(true); + await use({ modelName, createdModelId }); + } finally { + await deleteDeployment(page, createdModelId); + } + }, +}); + +test.describe("Edit LiteLLM Params on a deployment", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("params added on a deployment can be re-edited, and the deployment keeps serving", async ({ + page, + request, + deployment: { modelName, createdModelId }, + }) => { + await navigateToPage(page, Page.Models); + const modelIdCell = page.getByTestId(`model-id-${createdModelId}`); + await expect( + modelIdCell, + `the Models table lists ${modelName}`, + ).toBeVisible({ timeout: 15_000 }); + await modelIdCell.click(); + await expect(page.getByText("Back to Models").first()).toBeVisible({ + timeout: 15_000, + }); + + await editParams(page, (params) => ({ + ...params, + temperature: 0.2, + [CUSTOM_PARAM]: CUSTOM_PARAM_VALUE, + })); + const firstSave = await captureRequestBody( + page, + { method: "PATCH", urlIncludes: `/model/${createdModelId}/update` }, + async () => { + await page.getByRole("button", { name: "Save Changes" }).click(); + }, + ); + expect( + firstSave.litellm_params?.temperature, + "the added temperature goes on the wire", + ).toBe(0.2); + expect( + firstSave.litellm_params?.[CUSTOM_PARAM], + `the added ${CUSTOM_PARAM} goes on the wire`, + ).toEqual(CUSTOM_PARAM_VALUE); + expect( + firstSave.litellm_params?.model, + "a params edit does not rewrite the upstream model", + ).toBe(`openai/${modelName}`); + expect( + firstSave.litellm_params?.api_base, + "a params edit does not rewrite the api base", + ).toBe(MOCK_LLM_BASE); + expect( + firstSave.litellm_params, + "the credential is never re-sent, so a masked placeholder cannot overwrite the stored key", + ).not.toHaveProperty("api_key"); + + await expect + .poll( + async () => (await readStoredParams(page, createdModelId)).temperature, + { + message: "the added temperature never reached the stored deployment", + timeout: 20_000, + }, + ) + .toBe(0.2); + const afterFirstSave = await readStoredParams(page, createdModelId); + expect( + afterFirstSave[CUSTOM_PARAM], + `the added ${CUSTOM_PARAM} reached the stored deployment`, + ).toEqual(CUSTOM_PARAM_VALUE); + expect( + afterFirstSave.model, + "the stored upstream model survived the edit", + ).toBe(`openai/${modelName}`); + expect( + afterFirstSave.api_base, + "the stored api base survived the edit", + ).toBe(MOCK_LLM_BASE); + + await editParams(page, (params) => ({ + ...Object.fromEntries( + Object.entries(params).filter(([key]) => key !== CUSTOM_PARAM), + ), + temperature: 0.7, + })); + const secondSave = await captureRequestBody( + page, + { method: "PATCH", urlIncludes: `/model/${createdModelId}/update` }, + async () => { + await page.getByRole("button", { name: "Save Changes" }).click(); + }, + ); + expect( + secondSave.litellm_params?.temperature, + "a param set by an earlier save can be edited again", + ).toBe(0.7); + expect( + secondSave.litellm_params, + `dropping ${CUSTOM_PARAM} from the editor drops it from the request the UI sends`, + ).not.toHaveProperty(CUSTOM_PARAM); + expect( + secondSave.litellm_params?.model, + "a second params edit still leaves the upstream model alone", + ).toBe(`openai/${modelName}`); + expect( + secondSave.litellm_params?.api_base, + "a second params edit still leaves the api base alone", + ).toBe(MOCK_LLM_BASE); + expect( + secondSave.litellm_params, + "the credential is still never re-sent", + ).not.toHaveProperty("api_key"); + + await expect + .poll( + async () => (await readStoredParams(page, createdModelId)).temperature, + { + message: + "the re-edited temperature never reached the stored deployment", + timeout: 20_000, + }, + ) + .toBe(0.7); + + await page.reload(); + await expect( + page + .getByRole("tabpanel", { name: "Overview" }) + .getByText('"temperature": 0.7'), + "reopening the deployment renders the re-edited value, not the one from the first save", + ).toBeVisible({ timeout: 20_000 }); + + await sendChatCompletion(request, { + model: modelName, + prompt: `still serving ${modelName}`, + }); + }); +}); diff --git a/tests/e2e/ui/tests/modelsPage/modelHealthStatus.spec.ts b/tests/e2e/ui/tests/modelsPage/modelHealthStatus.spec.ts new file mode 100644 index 00000000000..247cce1b85d --- /dev/null +++ b/tests/e2e/ui/tests/modelsPage/modelHealthStatus.spec.ts @@ -0,0 +1,245 @@ +import { + test as base, + expect, + type Locator, + type Page as PlaywrightPage, +} from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; +import { readBack } from "../../helpers/roundTrip"; +import { masterKey } from "../../helpers/traffic"; + +const MOCK_LLM_BASE = `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`; +const UNREACHABLE_BASE = "http://127.0.0.1:9/v1"; + +async function isRegistered( + page: PlaywrightPage, + modelName: string, +): Promise { + const body = await readBack<{ data: { model_name?: string }[] }>( + page, + "/v2/model/info", + ); + return body.data.some((row) => row.model_name === modelName); +} + +function healthRow(page: PlaywrightPage, modelName: string): Locator { + return page.getByRole("row").filter({ hasText: modelName }); +} + +function pageOf(label: string): { current: number; total: number } { + const [current, total] = label + .replace("Page ", "") + .split(" of ") + .map((part) => Number(part.trim())); + return { current, total }; +} + +async function locateHealthRow( + page: PlaywrightPage, + modelName: string, +): Promise { + const pageLabel = page.getByTestId("pagination-page"); + await expect( + pageLabel, + "the health table reports which page it is showing", + ).toBeVisible({ timeout: 20_000 }); + + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + const row = healthRow(page, modelName); + const onThisPage = await row + .first() + .waitFor({ state: "visible", timeout: 3_000 }) + .then(() => true) + .catch(() => false); + if (onThisPage) return row; + + const { current, total } = pageOf(await pageLabel.innerText()); + const goTo = current < total ? current + 1 : 1; + if (total === 1) continue; + await page + .getByRole("button", { + name: current < total ? "Go to next page" : "Go to first page", + }) + .click(); + await expect(pageLabel).toContainText(`Page ${goTo} of`, { + timeout: 15_000, + }); + } + return healthRow(page, modelName); +} + +async function openHealthTab(page: PlaywrightPage): Promise { + await page.getByRole("tab", { name: "Health Status" }).click(); + await expect( + page.getByRole("heading", { name: "Model Health Status" }), + ).toBeVisible({ timeout: 15_000 }); +} + +async function expectStatus( + page: PlaywrightPage, + modelName: string, + status: string, +): Promise { + const row = await locateHealthRow(page, modelName); + await expect(row, `${modelName} has one row in the health table`).toHaveCount( + 1, + { timeout: 20_000 }, + ); + await expect( + row.getByText(status, { exact: true }), + `the Health Status cell for ${modelName} reads ${status}`, + ).toHaveCount(1, { timeout: 60_000 }); +} + +async function deleteDeployment( + page: PlaywrightPage, + id: string, +): Promise { + const post = () => + page.request.post("/model/delete", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { id }, + }); + const deleted = await post().catch(() => post()); + expect( + deleted.ok(), + `cleanup: /model/delete ${id} returned ${deleted.status()}`, + ).toBe(true); +} + +const uniqueSuffix = (): string => + `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + +async function withDeployment( + page: PlaywrightPage, + prefix: string, + apiBase: string, + use: (name: string) => Promise, +): Promise { + const name = `${prefix}-${uniqueSuffix()}`; + const created = await page.request.post("/model/new", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { + model_name: name, + litellm_params: { + model: `openai/${name}`, + api_base: apiBase, + api_key: "fake-key", + }, + model_info: {}, + }, + }); + expect( + created.ok(), + `/model/new for ${name} failed: ${created.status()} ${await created.text()}`, + ).toBe(true); + const id = (await created.json()).model_info?.id; + expect(id, `model id from /model/new for ${name}`).toBeTruthy(); + try { + await expect + .poll(() => isRegistered(page, name), { + message: `deployment ${name} never appeared in /v2/model/info after create`, + timeout: 60_000, + }) + .toBe(true); + await use(name); + } finally { + await deleteDeployment(page, id); + } +} + +const test = base.extend<{ reachableName: string; unreachableName: string }>({ + reachableName: async ({ page }, use) => { + await withDeployment(page, "e2e-health-up", MOCK_LLM_BASE, use); + }, + unreachableName: async ({ page }, use) => { + await withDeployment(page, "e2e-health-down", UNREACHABLE_BASE, use); + }, +}); + +test.describe("Model health status", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Run Health Check reports a reachable deployment healthy and an unreachable one unhealthy", async ({ + page, + reachableName, + unreachableName, + }) => { + await navigateToPage(page, Page.Models); + await openHealthTab(page); + + for (const name of [reachableName, unreachableName]) { + const row = await locateHealthRow(page, name); + await expect(row, `${name} has one row in the health table`).toHaveCount( + 1, + { timeout: 20_000 }, + ); + await row + .getByRole("button", { name: "Run Health Check", exact: true }) + .click(); + } + + await expectStatus(page, reachableName, "healthy"); + await expect( + healthRow(page, reachableName).getByText("unhealthy", { exact: true }), + "a reachable deployment is never reported unhealthy", + ).toHaveCount(0); + await expectStatus(page, unreachableName, "unhealthy"); + + const successDetail = ( + await locateHealthRow(page, reachableName) + ).getByRole("button", { + name: "View response details", + }); + await expect( + successDetail, + `${reachableName} offers its health check response for inspection`, + ).toBeVisible({ timeout: 60_000 }); + await successDetail.click(); + const successDialog = page.getByRole("dialog"); + await expect( + successDialog.getByRole("heading", { + name: `Health Check Response - ${reachableName}`, + }), + "the healthy deployment's detail opens its own response dialog", + ).toBeVisible({ timeout: 10_000 }); + await successDialog.getByRole("button", { name: "Close" }).last().click(); + await expect(successDialog).toBeHidden({ timeout: 10_000 }); + + const errorDetail = ( + await locateHealthRow(page, unreachableName) + ).getByRole("button", { + name: "View full error details", + }); + await expect( + errorDetail, + `${unreachableName} offers its health check error for inspection`, + ).toBeVisible({ timeout: 60_000 }); + await errorDetail.click(); + const errorDialog = page.getByRole("dialog"); + await expect( + errorDialog.getByRole("heading", { + name: `Health Check Error - ${unreachableName}`, + }), + "the unreachable deployment's detail opens its own error dialog", + ).toBeVisible({ timeout: 10_000 }); + await expect( + errorDialog, + "the error dialog carries the upstream connection failure, not a generic message", + ).toContainText(/connection error/i, { timeout: 10_000 }); + await expect( + errorDialog, + "the error dialog names the endpoint that could not be reached", + ).toContainText(UNREACHABLE_BASE); + await errorDialog.getByRole("button", { name: "Close" }).last().click(); + await expect(errorDialog).toBeHidden({ timeout: 10_000 }); + + await page.reload(); + await openHealthTab(page); + await expectStatus(page, reachableName, "healthy"); + await expectStatus(page, unreachableName, "unhealthy"); + }); +}); From b3151073d2a8ee274fd067c1d4cfb17da0a82459 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 8 Sep 2026 22:49:54 -0700 Subject: [PATCH 175/411] test(e2e/ui): cover key budget window, non-admin model scope edit, and key blocking (#40027) * test(e2e/ui): cover key budget window, non-admin model scope edit, and key blocking Three Playwright specs for the Virtual Keys flows customers hit most, each reading its result back through /key/info and /v1/chat/completions rather than trusting the toast: - a monthly spend cap and reset window set through Edit Settings, surviving a reload, with clearing the window leaving the cap in place - a team member narrowing their own team key's models, and the proxy refusing the model they dropped - blocking a key from its detail page, then unblocking it Each test owns the key it edits and deletes it on teardown, so retries and --repeat-each never run out of fixtures. * test(e2e/ui): tighten virtual key specs from review feedback Replace the mutable suite-level key state with a Playwright fixture, so the alias and token are never reassigned and cleanup stays tied to the test. Assert /key/delete succeeded instead of discarding the response, so a failed cleanup surfaces rather than leaving rows behind. Drop the explanatory JSDoc the repo's comment policy disallows, keeping only the one line explaining why Date.now() alone is not unique enough. Type the master-key POST helper against a real guard instead of casting to Record. Assert the unblocked key is served with a 200, not just the response text, and that clearing the reset window also clears budget_reset_at. * test(ui): assert the team response through Playwright --- tests/e2e/ui/helpers/navigation.ts | 10 + tests/e2e/ui/helpers/traffic.ts | 51 ++++- .../internalUserKeyScope.spec.ts | 208 ++++++++++++++++++ .../ui/tests/proxy-admin/keyBlocking.spec.ts | 112 ++++++++++ .../tests/proxy-admin/keyBudgetWindow.spec.ts | 101 +++++++++ 5 files changed, 478 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/ui/tests/internal-user/internalUserKeyScope.spec.ts create mode 100644 tests/e2e/ui/tests/proxy-admin/keyBlocking.spec.ts create mode 100644 tests/e2e/ui/tests/proxy-admin/keyBudgetWindow.spec.ts diff --git a/tests/e2e/ui/helpers/navigation.ts b/tests/e2e/ui/helpers/navigation.ts index 4a7c4e7baa9..e0e7b4da396 100644 --- a/tests/e2e/ui/helpers/navigation.ts +++ b/tests/e2e/ui/helpers/navigation.ts @@ -73,3 +73,13 @@ export async function clickTeamId(page: PlaywrightPage, teamId: string): Promise await cell.click(); await expect(page.getByText("Back to Teams")).toBeVisible({ timeout: 10_000 }); } + +export async function openKeyDetail(page: PlaywrightPage, alias: string): Promise { + await page.getByPlaceholder("Search by key alias or ID").fill(alias); + const row = page.getByRole("row").filter({ hasText: alias }); + await expect(row, `key row "${alias}" never appeared on the Virtual Keys page`).toBeVisible({ timeout: 15_000 }); + await row.getByRole("button", { name: alias }).click(); + await expect(page.getByText("Back to Keys"), `key detail for "${alias}" never opened`).toBeVisible({ + timeout: 15_000, + }); +} diff --git a/tests/e2e/ui/helpers/traffic.ts b/tests/e2e/ui/helpers/traffic.ts index 7f8417cdffb..cb68747b364 100644 --- a/tests/e2e/ui/helpers/traffic.ts +++ b/tests/e2e/ui/helpers/traffic.ts @@ -1,4 +1,4 @@ -import { APIRequestContext, expect } from "@playwright/test"; +import { APIRequestContext, APIResponse, expect } from "@playwright/test"; /** Model names served by fixtures/config.yml, both backed by the mock LLM server. */ export const CHAT_MODEL_A = "fake-openai-gpt-4"; @@ -15,6 +15,9 @@ export const masterKey = (): string => process.env.LITELLM_MASTER_KEY || "sk-123 export const rootPath = (): string => process.env.SERVER_ROOT_PATH ?? ""; +/** Date.now() alone collides: `--repeat-each` starts its copies inside the same millisecond. */ +export const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + interface ChatOptions { model: string; prompt: string; @@ -25,9 +28,8 @@ interface ChatOptions { traceId?: string; } -/** POST /v1/chat/completions and return the completion id (the Logs Request ID). */ -export async function sendChatCompletion(request: APIRequestContext, opts: ChatOptions): Promise { - const res = await request.post(`${rootPath()}/v1/chat/completions`, { +const postChatCompletion = (request: APIRequestContext, opts: ChatOptions): Promise => + request.post(`${rootPath()}/v1/chat/completions`, { headers: { Authorization: `Bearer ${opts.apiKey ?? masterKey()}`, "Content-Type": "application/json", @@ -39,12 +41,26 @@ export async function sendChatCompletion(request: APIRequestContext, opts: ChatO ...(opts.traceId ? { litellm_trace_id: opts.traceId } : {}), }, }); + +/** POST /v1/chat/completions and return the completion id (the Logs Request ID). */ +export async function sendChatCompletion(request: APIRequestContext, opts: ChatOptions): Promise { + const res = await postChatCompletion(request, opts); expect(res.ok(), `chat completion for ${opts.model} failed (${res.status()}): ${await res.text()}`).toBe(true); const body = await res.json(); expect(body.choices?.[0]?.message?.content).toContain(MOCK_RESPONSE_TEXT); return body.id as string; } +export interface ChatAttempt { + status: number; + body: string; +} + +export async function attemptChatCompletion(request: APIRequestContext, opts: ChatOptions): Promise { + const res = await postChatCompletion(request, opts); + return { status: res.status(), body: await res.text() }; +} + /** `key` is the sk- value to authenticate with; `token` is its hash, which spend aggregates are keyed by. */ export async function createVirtualKey( request: APIRequestContext, @@ -66,6 +82,33 @@ export async function createVirtualKey( }; } +export interface KeyInfo { + key_alias: string | null; + max_budget: number | null; + budget_duration: string | null; + budget_reset_at: string | null; + blocked: boolean | null; + models: string[]; + team_id: string | null; +} + +export async function readKeyInfo(request: APIRequestContext, token: string): Promise { + const res = await request.get(`${rootPath()}/key/info?key=${encodeURIComponent(token)}`, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + expect(res.ok(), `GET /key/info for ${token} failed (${res.status()}): ${await res.text()}`).toBe(true); + const body = await res.json(); + return body.info as KeyInfo; +} + +export async function deleteVirtualKey(request: APIRequestContext, token: string): Promise { + const res = await request.post(`${rootPath()}/key/delete`, { + headers: { Authorization: `Bearer ${masterKey()}`, "Content-Type": "application/json" }, + data: { keys: [token] }, + }); + expect(res.ok(), `key delete for ${token} failed (${res.status()}): ${await res.text()}`).toBe(true); +} + /** Spend logs are flushed on a timer, so an assertion straight after a completion races the writer. */ export async function waitForSpendLog( request: APIRequestContext, diff --git a/tests/e2e/ui/tests/internal-user/internalUserKeyScope.spec.ts b/tests/e2e/ui/tests/internal-user/internalUserKeyScope.spec.ts new file mode 100644 index 00000000000..f923841257a --- /dev/null +++ b/tests/e2e/ui/tests/internal-user/internalUserKeyScope.spec.ts @@ -0,0 +1,208 @@ +import { test, expect, type APIRequestContext } from "@playwright/test"; +import { Page } from "../../fixtures/pages"; +import { + dismissFeedbackPopup, + navigateToPage, + openKeyDetail, +} from "../../helpers/navigation"; +import { + CHAT_MODEL_A, + CHAT_MODEL_B, + MOCK_RESPONSE_TEXT, + attemptChatCompletion, + createVirtualKey, + deleteVirtualKey, + masterKey, + readKeyInfo, + rootPath, + uniqueSuffix, +} from "../../helpers/traffic"; + +const MEMBER_PASSWORD = "E2e-Team-Member-Pass-1!"; + +interface CreatedTeam { + readonly team_id: string; +} + +function assertCreatedTeam(body: unknown): asserts body is CreatedTeam { + expect(body, "/team/new returned no team_id").toMatchObject({ + team_id: expect.any(String), + }); +} + +async function postAsMaster( + request: APIRequestContext, + path: string, + data: Record, +): Promise { + const res = await request.post(`${rootPath()}${path}`, { + headers: { + Authorization: `Bearer ${masterKey()}`, + "Content-Type": "application/json", + }, + data, + }); + expect( + res.ok(), + `POST ${path} failed (${res.status()}): ${await res.text()}`, + ).toBe(true); + return res.json(); +} + +test.describe("Internal User - own team key model scope", () => { + test.use({ storageState: { cookies: [], origins: [] } }); + + test("a team member narrows their own key's models and the proxy enforces it", async ({ + page, + request, + }) => { + const suffix = uniqueSuffix(); + const email = `team-member-${suffix}@test.local`; + const userId = `e2e-key-scope-user-${suffix}`; + const alias = `e2e-key-scope-${suffix}`; + + const team = await postAsMaster(request, "/team/new", { + team_alias: `E2E Key Scope ${suffix}`, + models: [CHAT_MODEL_A, CHAT_MODEL_B], + team_member_permissions: ["/key/generate", "/key/update", "/key/info"], + }); + assertCreatedTeam(team); + const teamId = team.team_id; + + try { + await postAsMaster(request, "/user/new", { + user_id: userId, + user_email: email, + user_role: "internal_user", + auto_create_key: false, + }); + await postAsMaster(request, "/user/update", { + user_id: userId, + password: MEMBER_PASSWORD, + }); + await postAsMaster(request, "/team/member_add", { + team_id: teamId, + member: { role: "user", user_id: userId }, + }); + + const created = await createVirtualKey(request, { + key_alias: alias, + team_id: teamId, + user_id: userId, + models: [], + }); + + try { + await page.goto("/ui/login"); + await page.getByPlaceholder("Enter your username").fill(email); + await page + .getByPlaceholder("Enter your password") + .fill(MEMBER_PASSWORD); + await page.getByRole("button", { name: "Login", exact: true }).click(); + await expect( + page.locator("a", { hasText: "Virtual Keys" }), + `${email} never reached the dashboard`, + ).toBeVisible({ timeout: 30_000 }); + await dismissFeedbackPopup(page); + + await navigateToPage(page, Page.ApiKeys); + await openKeyDetail(page, alias); + + await page.getByRole("tab", { name: "Settings" }).click(); + await page.getByRole("button", { name: "Edit Settings" }).click(); + + await page.getByRole("combobox", { name: "Select models" }).click(); + await expect( + page.getByRole("option", { name: CHAT_MODEL_A, exact: true }), + `the Models dropdown does not offer ${CHAT_MODEL_A} to a team member`, + ).toBeVisible({ timeout: 15_000 }); + await expect( + page.getByRole("option", { name: CHAT_MODEL_B, exact: true }), + `the Models dropdown does not offer ${CHAT_MODEL_B} to a team member`, + ).toBeVisible(); + + await page + .getByRole("option", { name: CHAT_MODEL_A, exact: true }) + .click(); + await page.keyboard.press("Escape"); + + const updated = page.waitForResponse( + (res) => + res.url().includes("/key/update") && + res.request().method() === "POST", + ); + await page.getByRole("button", { name: "Save Changes" }).click(); + const updateStatus = (await updated).status(); + expect( + updateStatus, + "a team member's own-key edit was refused", + ).toBeGreaterThanOrEqual(200); + expect( + updateStatus, + "a team member's own-key edit was refused", + ).toBeLessThan(300); + await expect( + page.getByText("Key updated successfully").first(), + ).toBeVisible({ timeout: 15_000 }); + + await expect + .poll( + async () => (await readKeyInfo(request, created.token)).models, + { + message: `the narrowed model scope never reached /key/info for ${alias}`, + timeout: 20_000, + }, + ) + .toEqual([CHAT_MODEL_A]); + + await expect + .poll( + async () => + await attemptChatCompletion(request, { + model: CHAT_MODEL_B, + prompt: `out of scope ${suffix}`, + apiKey: created.key, + }), + { + message: `${CHAT_MODEL_B} was still served after the key was narrowed to ${CHAT_MODEL_A}`, + timeout: 30_000, + }, + ) + .toMatchObject({ + status: 403, + body: expect.stringContaining(CHAT_MODEL_B), + }); + + const inScope = await attemptChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `in scope ${suffix}`, + apiKey: created.key, + }); + expect( + inScope, + `${CHAT_MODEL_A} is no longer served by the narrowed key`, + ).toMatchObject({ + status: 200, + body: expect.stringContaining(MOCK_RESPONSE_TEXT), + }); + } finally { + await deleteVirtualKey(request, created.token); + } + } finally { + await request.post(`${rootPath()}/user/delete`, { + headers: { + Authorization: `Bearer ${masterKey()}`, + "Content-Type": "application/json", + }, + data: { user_ids: [userId] }, + }); + await request.post(`${rootPath()}/team/delete`, { + headers: { + Authorization: `Bearer ${masterKey()}`, + "Content-Type": "application/json", + }, + data: { team_ids: [teamId] }, + }); + } + }); +}); diff --git a/tests/e2e/ui/tests/proxy-admin/keyBlocking.spec.ts b/tests/e2e/ui/tests/proxy-admin/keyBlocking.spec.ts new file mode 100644 index 00000000000..99a8065a797 --- /dev/null +++ b/tests/e2e/ui/tests/proxy-admin/keyBlocking.spec.ts @@ -0,0 +1,112 @@ +import { test as base, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { dismissFeedbackPopup, navigateToPage, openKeyDetail } from "../../helpers/navigation"; +import { + CHAT_MODEL_A, + MOCK_RESPONSE_TEXT, + attemptChatCompletion, + createVirtualKey, + deleteVirtualKey, + readKeyInfo, + sendChatCompletion, + uniqueSuffix, +} from "../../helpers/traffic"; + +interface ScopedKey { + alias: string; + token: string; + apiKey: string; +} + +const test = base.extend<{ scopedKey: ScopedKey }>({ + scopedKey: async ({ page }, use) => { + const alias = `e2e-block-key-${uniqueSuffix()}`; + const created = await createVirtualKey(page.request, { + key_alias: alias, + models: [CHAT_MODEL_A], + }); + await use({ alias, token: created.token, apiKey: created.key }); + await deleteVirtualKey(page.request, created.token); + }, +}); + +test.describe("Proxy Admin - Key blocking", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("blocking a key stops it serving and unblocking restores it", async ({ page, scopedKey }) => { + const { alias, token, apiKey } = scopedKey; + + await sendChatCompletion(page.request, { + model: CHAT_MODEL_A, + prompt: `pre-block ${alias}`, + apiKey, + }); + + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + await openKeyDetail(page, alias); + + await page.getByRole("button", { name: "More key actions" }).click(); + await page.getByRole("menuitem", { name: "Block Key" }).click(); + const blockDialog = page.getByRole("dialog", { name: "Block Key" }); + await expect(blockDialog, "the Block Key confirmation never opened").toBeVisible({ timeout: 10_000 }); + await blockDialog.getByRole("button", { name: "Block", exact: true }).click(); + + await expect + .poll(async () => (await readKeyInfo(page.request, token)).blocked, { + message: "the key never came back blocked from /key/info", + timeout: 20_000, + }) + .toBe(true); + + await expect + .poll( + async () => + await attemptChatCompletion(page.request, { + model: CHAT_MODEL_A, + prompt: "blocked", + apiKey, + }), + { + message: "a blocked key was still served by /v1/chat/completions", + timeout: 30_000, + }, + ) + .toMatchObject({ status: 401, body: expect.stringContaining("blocked") }); + + await page.reload(); + await expect( + page.getByText("Blocked", { exact: true }), + "the reloaded key detail does not show the key as blocked", + ).toBeVisible({ timeout: 15_000 }); + + await page.getByRole("button", { name: "More key actions" }).click(); + await page.getByRole("menuitem", { name: "Unblock Key" }).click(); + const unblockDialog = page.getByRole("dialog", { name: "Unblock Key" }); + await expect(unblockDialog, "the Unblock Key confirmation never opened").toBeVisible({ timeout: 10_000 }); + await unblockDialog.getByRole("button", { name: "Unblock", exact: true }).click(); + + await expect + .poll(async () => (await readKeyInfo(page.request, token)).blocked, { + message: "the key never came back unblocked from /key/info", + timeout: 20_000, + }) + .toBe(false); + + await expect + .poll( + async () => + await attemptChatCompletion(page.request, { + model: CHAT_MODEL_A, + prompt: "unblocked", + apiKey, + }), + { + message: "an unblocked key is still refused by /v1/chat/completions", + timeout: 30_000, + }, + ) + .toMatchObject({ status: 200, body: expect.stringContaining(MOCK_RESPONSE_TEXT) }); + }); +}); diff --git a/tests/e2e/ui/tests/proxy-admin/keyBudgetWindow.spec.ts b/tests/e2e/ui/tests/proxy-admin/keyBudgetWindow.spec.ts new file mode 100644 index 00000000000..4e4d0a395c3 --- /dev/null +++ b/tests/e2e/ui/tests/proxy-admin/keyBudgetWindow.spec.ts @@ -0,0 +1,101 @@ +import { test as base, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH, E2E_TEAM_CRUD_ID } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { dismissFeedbackPopup, navigateToPage, openKeyDetail } from "../../helpers/navigation"; +import { captureRequestBody } from "../../helpers/roundTrip"; +import { CHAT_MODEL_A, createVirtualKey, deleteVirtualKey, readKeyInfo, uniqueSuffix } from "../../helpers/traffic"; + +interface ScopedKey { + alias: string; + token: string; +} + +const test = base.extend<{ scopedKey: ScopedKey }>({ + scopedKey: async ({ page }, use) => { + const alias = `e2e-budget-window-${uniqueSuffix()}`; + const created = await createVirtualKey(page.request, { + key_alias: alias, + team_id: E2E_TEAM_CRUD_ID, + models: [CHAT_MODEL_A], + }); + await use({ alias, token: created.token }); + await deleteVirtualKey(page.request, created.token); + }, +}); + +test.describe("Proxy Admin - Key budget window", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("a monthly spend cap survives a reload, and clearing the window keeps the cap", async ({ page, scopedKey }) => { + const { alias, token } = scopedKey; + + const before = await readKeyInfo(page.request, token); + expect(before.max_budget, "a freshly generated key starts with no budget").toBeNull(); + + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + await openKeyDetail(page, alias); + + await page.getByRole("tab", { name: "Settings" }).click(); + await page.getByRole("button", { name: "Edit Settings" }).click(); + + await page.getByRole("spinbutton", { name: "Max Budget (USD)" }).fill("12.5"); + await page.getByLabel("Reset Budget", { exact: true }).click(); + await page.getByRole("option", { name: "monthly", exact: true }).click(); + await page.getByRole("button", { name: "Save Changes" }).click(); + + await expect + .poll(async () => (await readKeyInfo(page.request, token)).max_budget, { + message: "the $12.50 cap never reached /key/info", + timeout: 20_000, + }) + .toBe(12.5); + await expect + .poll(async () => (await readKeyInfo(page.request, token)).budget_duration, { + message: "the monthly reset window never reached /key/info", + timeout: 20_000, + }) + .toBe("30d"); + + const capped = await readKeyInfo(page.request, token); + const resetAt = new Date(capped.budget_reset_at ?? ""); + expect(Number.isNaN(resetAt.getTime()), "a monthly window left the key with no budget_reset_at").toBe(false); + expect(resetAt.getTime(), "budget_reset_at was set in the past").toBeGreaterThan(Date.now()); + expect(resetAt.getUTCDate(), "a monthly window resets on the 1st, a daily one would not").toBe(1); + + await page.reload(); + await expect( + page.getByRole("paragraph").filter({ hasText: "of $12.50" }), + "the reloaded key detail does not render the $12.50 cap", + ).toBeVisible({ timeout: 15_000 }); + + await page.getByRole("tab", { name: "Settings" }).click(); + await expect( + page.getByTestId("budget-reset-value"), + "the reloaded key detail does not name the 30d reset window", + ).toHaveText(/Every 30d/, { timeout: 15_000 }); + + await page.getByRole("button", { name: "Edit Settings" }).click(); + await page.getByLabel("Reset Budget", { exact: true }).click(); + await page.getByRole("option", { name: "Never resets", exact: true }).click(); + + const cleared = await captureRequestBody(page, { method: "POST", urlIncludes: "/key/update" }, async () => { + await page.getByRole("button", { name: "Save Changes" }).click(); + }); + expect(cleared).toHaveProperty("budget_duration"); + expect(cleared.budget_duration, "clearing the window must send budget_duration: null explicitly").toBeNull(); + + await expect + .poll(async () => (await readKeyInfo(page.request, token)).budget_duration, { + message: "the reset window was never cleared on /key/info", + timeout: 20_000, + }) + .toBeNull(); + + const after = await readKeyInfo(page.request, token); + expect(after.budget_reset_at, "clearing the reset window left a stale next-reset timestamp").toBeNull(); + expect(after.max_budget, "clearing the reset window also wiped the spend cap").toBe(12.5); + expect(after.models, "editing the budget left the key's models untouched").toEqual(before.models); + expect(after.team_id, "editing the budget left the key's team untouched").toEqual(before.team_id); + }); +}); From 36bd7f113837caadd3f3d400dd246cd4649b7b33 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 8 Sep 2026 22:50:13 -0700 Subject: [PATCH 176/411] fix(mcp): honor an explicit null on toolset update, cover MCP lifecycle e2e (#40022) * fix(mcp): honor an explicit null on toolset update, cover MCP lifecycle e2e PUT /v1/mcp/toolset dumped its payload with exclude_none, so a field sent as null looked exactly like one the caller left out and the stored value survived. An admin could not clear a toolset's description: the save reported success and the old text came straight back. It now dumps with exclude_unset, so absent keeps and null clears, which is what PUT /v1/mcp/server already did. A null tools list clears the selection to empty, and a null toolset_name is ignored because a toolset always has a name. Adds create, read, partial-update, clear and delete e2e coverage for MCP servers and toolsets, with every read-back polled on every replica so an edit that lands on one replica and not another fails the test, plus an enforcement test proving a key granted a toolset lists exactly that toolset's tools against the real Datadog upstream. * fix(e2e): refuse a read-back that no replica serves A read-back over an empty replica mapping satisfied every predicate and returned as if it had converged, so it would have asserted nothing and passed. No wiring can produce that today, since the replica list always falls back to at least one URL, but a helper whose whole job is proving a write reached every replica should not have a shape that passes vacuously. * fix(mcp): keep a null tools list a no-op on toolset update Treating a null tools list as a clear meant an existing client that sends tools=null during a partial update, meaning "leave the selection alone", silently lost every tool the toolset grants. That is a permission surface, so the quiet version of it is the worst version. A toolset always has a tool list, the same way it always has a name, so a null on either is now a no-op. Emptying the selection is an explicit [], which cannot be confused with a field the caller left out, and which is what the dashboard already sends. * fix(e2e): keep MCP admin routes on the data plane /v1/mcp/* is a lazily mounted feature, so a gateway registers it on the first matching request, which happens after the startup route trim that drops management endpoints. Routing it to the control plane therefore sent every MCP call to the one backend process: the new lifecycle read-backs proved a single process rather than every replica, and mcp_client's await_registered barrier waited on a registry that does not serve the tools/list call it guards, so the existing MCP suites polled a gateway that had not synced yet until poll_timeout Verified against a two-gateway split stack (backend on 4001, gateways on 4010 and 4011, one postgres): both gateways answer /v1/mcp/server and /v1/mcp/toolset, and each served 6 server reads and 7 toolset reads over the run * fix(e2e): grant the toolset by the tool's own name, not the wire name tools/list serves a tool as , but a toolset grants by the tool's own name: resolve_toolset_permissions reads toolset.tools[].tool_name straight through, and the prefix is added on the way out. The test built the toolset from the names tools/list reported, so the grant matched nothing, the scoped key listed no tools, and await_tools ran out its whole poll_timeout before failing Measure the prefix off search_datadog_logs, whose own name is known, rather than guessing it from the alias, since the proxy can be configured to prefix with a short server id instead. The expectation compared against tools/list stays in wire names; only what the toolset stores crosses back * test(mcp): build immutable lifecycle updates and replica results * test: validate opaque stream IDs and hide log-reader credentials * test: isolate auto-router scenarios and clean partial setup * test: honor Datadog search rate-limit reset headers * test: share the Datadog read-back deadline across retries * test: preserve captured MCP toolset update fields --- .../_experimental/mcp_server/toolset_db.py | 16 +- .../mcp_management_endpoints.py | 4 + tests/e2e/coverage_registry/mcp.yaml | 8 + tests/e2e/coverage_registry/mgmt.yaml | 10 + tests/e2e/e2e_http.py | 64 +++- .../test_responses_bridge_streaming_e2e.py | 24 +- tests/e2e/logging/datadog_reader.py | 124 +++++--- tests/e2e/logging/test_datadog_reader.py | 223 +++++++++++++ tests/e2e/management/management_client.py | 35 +++ .../e2e/management/test_mcp_lifecycle_e2e.py | 294 ++++++++++++++++++ tests/e2e/mcp/datadog_mcp.py | 7 +- tests/e2e/mcp/mcp_client.py | 39 ++- .../mcp/test_mcp_toolset_enforcement_e2e.py | 95 ++++++ tests/e2e/models.py | 73 ++++- tests/e2e/proxy_client.py | 216 ++++++++++++- .../test_auto_router_regressions_e2e.py | 239 +++++++------- tests/e2e/test_e2e_http.py | 91 +++++- tests/e2e/test_proxy_client.py | 95 +++++- .../mcp_server/test_mcp_partial_update.py | 71 ++++- 19 files changed, 1501 insertions(+), 227 deletions(-) create mode 100644 tests/e2e/logging/test_datadog_reader.py create mode 100644 tests/e2e/management/test_mcp_lifecycle_e2e.py create mode 100644 tests/e2e/mcp/test_mcp_toolset_enforcement_e2e.py diff --git a/litellm/proxy/_experimental/mcp_server/toolset_db.py b/litellm/proxy/_experimental/mcp_server/toolset_db.py index ecaaf35e817..48bad178927 100644 --- a/litellm/proxy/_experimental/mcp_server/toolset_db.py +++ b/litellm/proxy/_experimental/mcp_server/toolset_db.py @@ -132,10 +132,18 @@ async def update_mcp_toolset( data: UpdateMCPToolsetRequest, touched_by: str, ) -> MCPToolset | None: - data_dict: Final = data.model_dump(exclude_none=True, exclude={"toolset_id"}) - if "tools" in data_dict: - data_dict["tools"] = json.dumps(data_dict["tools"]) - data_dict["updated_by"] = touched_by + """A partial update: absent keeps, null clears. A toolset always has a name and a + tool list, so a null ``toolset_name`` or ``tools`` is a no-op rather than a clear; + emptying the tool selection is an explicit ``[]``, which cannot be mistaken for a + caller that left the field out.""" + data_dict: Final = dict( # mutable-ok: Prisma requires a plain dict for JSON query serialization + ( + (field, json.dumps(value) if field == "tools" else value) + for field, value in data.model_dump(exclude_unset=True).items() + if field != "toolset_id" and (field not in ("toolset_name", "tools") or value is not None) + ), + updated_by=touched_by, + ) try: row: Final = await _toolset_table(prisma_client).update( where={"toolset_id": data.toolset_id}, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index d5c3427f29a..2aa7fdc7393 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -2673,6 +2673,8 @@ if MCP_AVAILABLE: """ Updates the MCP Server in the db. + Partial update: a field left out of the payload keeps its stored value, and a field sent as null is cleared. + Parameters: - payload: UpdateMCPServerRequest - Required. The updated mcp server data. ``` @@ -3098,6 +3100,8 @@ if MCP_AVAILABLE: user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), litellm_changed_by: str | None = Header(None), ): + """Partial update: a field left out keeps its stored value, and a field sent as null is cleared, except + ``toolset_name`` and ``tools``, which a toolset always has; empty the tool selection with an explicit [].""" prisma_client: Final = get_prisma_client_or_throw("Database not connected. Connect a database to your proxy") if LitellmUserRoles.PROXY_ADMIN != user_api_key_dict.user_role: raise HTTPException( diff --git a/tests/e2e/coverage_registry/mcp.yaml b/tests/e2e/coverage_registry/mcp.yaml index ab644118a47..f853e9ff8d6 100644 --- a/tests/e2e/coverage_registry/mcp.yaml +++ b/tests/e2e/coverage_registry/mcp.yaml @@ -119,3 +119,11 @@ assertions: [succeeds] source: "server.py:1089" rationale: Smoke; rarely used; same auth model as tools +- id: mcp.list_tools.api_key.toolset_scoped + module: mcp + tier: P0 + operation: list_tools + auth_family: api_key + assertions: [toolset_scoped] + source: "user_api_key_auth_mcp.py:2137" + rationale: "A key granted a toolset lists exactly the toolset's tools: the rest of the server's catalog stays hidden and every stored name resolves" diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 860d96a50b4..c8d7037d2fd 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -76,3 +76,13 @@ - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} - {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven} +- {id: mgmt.mcp_server.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1577", rationale: "Every field of an admin-created MCP server reads back verbatim, by id and in the list, on every replica"} +- {id: mgmt.mcp_server.list.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1112", rationale: "The MCP page grid lists a created server with the same field values its detail view reports"} +- {id: mgmt.mcp_server.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "mcp_management_endpoints.py:2665", rationale: "A dashboard edit of one field leaves the others intact and is visible on every replica after one save; edits that took several saves to stick were a customer defect"} +- {id: mgmt.mcp_server.update.clear_persists, module: mgmt, tier: P0, surface: api, assertions: [clear_persists], source: "mcp_management_endpoints.py:2665", rationale: "An explicit null clears the stored field (absent keeps, null clears)"} +- {id: mgmt.mcp_server.delete.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:2139", rationale: "A deleted server is gone by id and from the list on every replica"} +- {id: mgmt.mcp_toolset.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:3009", rationale: "Toolset tools read back under the exact server_id and tool_name written; a toolset stored under one name and read under another granted nothing"} +- {id: mgmt.mcp_toolset.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "mcp_management_endpoints.py:3098", rationale: "Editing the description leaves the tools and name intact"} +- {id: mgmt.mcp_toolset.update.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:3098", rationale: "Narrowing the tools to one entry reads back exactly that entry"} +- {id: mgmt.mcp_toolset.update.clear_persists, module: mgmt, tier: P0, surface: api, assertions: [clear_persists], source: "mcp_management_endpoints.py:3098", fail_before_fix: proven, rationale: "An explicit null clears the stored description; the update used to drop null and keep the old value"} +- {id: mgmt.mcp_toolset.delete.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:3149", rationale: "A deleted toolset is gone by id and from the list on every replica"} diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 9d5f1658e91..415c72bbb3c 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -49,6 +49,11 @@ class AnthropicHeaders(AuthHeaders): anthropic_version: str = Field(default="2023-06-01", alias="anthropic-version") +class PartialBody(BaseModel): + """A body for a partial-update route (absent = keep, null = clear): a field left + unset is omitted from the wire, and a field set to None is sent as JSON null.""" + + class NoBody(BaseModel): """Empty body/query for routes that take none.""" @@ -252,6 +257,13 @@ def assert_auth_denied(result: StreamingResponse, context: str) -> None: f"{context}: expected 401/403, got {result.status_code}: {result.body[:300]}" ) + +def wire_body(json: BaseModel) -> dict[str, object]: + if isinstance(json, PartialBody): + return json.model_dump(by_alias=True, exclude_unset=True) + return json.model_dump(by_alias=True, exclude_none=True) + + def _headers(headers: BaseModel) -> dict[str, str]: dumped: dict[str, object] = headers.model_dump(by_alias=True, exclude_none=True) return {key: str(value) for key, value in dumped.items()} @@ -307,9 +319,26 @@ def request_with_retry[T: RetryableResponse]( return issue() -def _classify[R: BaseModel]( - resp: requests.Response, response_type: type[R] -) -> Result[R]: +class ClassifiableResponse(Protocol): + """What classifying an outcome reads off a response. requests.Response satisfies + it, and so does a fake, so the classification rules are testable on their own.""" + + @property + def status_code(self) -> int: ... + + @property + def ok(self) -> bool: ... + + @property + def text(self) -> str: ... + + @property + def content(self) -> bytes: ... + + def json(self) -> object: ... + + +def classify[R: BaseModel](resp: ClassifiableResponse, response_type: type[R]) -> Result[R]: if resp.status_code == 401: return UnauthorizedError(body=resp.text) if resp.status_code == 429: @@ -317,7 +346,8 @@ def _classify[R: BaseModel]( if not resp.ok: return UnknownApiError(status_code=resp.status_code, body=resp.text) try: - return Success(status_code=resp.status_code, data=response_type.model_validate(resp.json())) + payload: Final[object] = resp.json() if resp.content else {} + return Success(status_code=resp.status_code, data=response_type.model_validate(payload)) except Exception as exc: # noqa: BLE001 - any parse/validation failure is a value return ValidationError(message=str(exc)) @@ -335,13 +365,13 @@ def post[R: BaseModel]( lambda: requests.post( str(url), headers=_headers(headers), - json=json.model_dump(by_alias=True, exclude_none=True), + json=wire_body(json), timeout=timeout, ) ) except requests.RequestException as exc: return NetworkError(message=str(exc)) - return _classify(resp, response_type) + return classify(resp, response_type) def get[R: BaseModel]( @@ -363,7 +393,7 @@ def get[R: BaseModel]( ) except requests.RequestException as exc: return NetworkError(message=str(exc)) - return _classify(resp, response_type) + return classify(resp, response_type) def get_external[R: BaseModel]( @@ -383,7 +413,7 @@ def get_external[R: BaseModel]( ) except requests.RequestException as exc: return NetworkError(message=str(exc)) - return _classify(resp, response_type) + return classify(resp, response_type) def delete[R: BaseModel]( @@ -400,14 +430,14 @@ def delete[R: BaseModel]( lambda: requests.delete( str(url), headers=_headers(headers), - json=json.model_dump(by_alias=True, exclude_none=True), + json=wire_body(json), params=_params(params), timeout=timeout, ) ) except requests.RequestException as exc: return NetworkError(message=str(exc)) - return _classify(resp, response_type) + return classify(resp, response_type) def patch[R: BaseModel]( @@ -423,13 +453,13 @@ def patch[R: BaseModel]( lambda: requests.patch( str(url), headers=_headers(headers), - json=json.model_dump(by_alias=True, exclude_none=True), + json=wire_body(json), timeout=timeout, ) ) except requests.RequestException as exc: return NetworkError(message=str(exc)) - return _classify(resp, response_type) + return classify(resp, response_type) def put[R: BaseModel]( @@ -445,13 +475,13 @@ def put[R: BaseModel]( lambda: requests.put( str(url), headers=_headers(headers), - json=json.model_dump(by_alias=True, exclude_none=True), + json=wire_body(json), timeout=timeout, ) ) except requests.RequestException as exc: return NetworkError(message=str(exc)) - return _classify(resp, response_type) + return classify(resp, response_type) def probe( @@ -555,7 +585,7 @@ def send( str(url), headers=_headers(headers), params=_params(params), - json=json.model_dump(by_alias=True, exclude_none=True), + json=wire_body(json), stream=stream, timeout=timeout, ) @@ -605,7 +635,7 @@ def upload[R: BaseModel]( ) except requests.RequestException as exc: return NetworkError(message=str(exc)) - return _classify(resp, response_type) + return classify(resp, response_type) def stream_binary( @@ -623,7 +653,7 @@ def stream_binary( resp = requests.post( str(url), headers=_headers(headers), - json=json.model_dump(by_alias=True, exclude_none=True), + json=wire_body(json), stream=True, timeout=timeout, ) diff --git a/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py b/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py index 9a45743a0cd..75817340876 100644 --- a/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py +++ b/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py @@ -16,8 +16,10 @@ into a chat completion chunk. Two customer-visible contracts only hold on that p from __future__ import annotations +from typing import Final, Literal + import pytest -from pydantic import BaseModel +from pydantic import BaseModel, Field from e2e_config import unique_marker from e2e_http import StreamingResponse @@ -51,7 +53,8 @@ class _BridgeChoice(BaseModel): class _BridgeChunk(BaseModel): id: str - choices: list[_BridgeChoice] = [] + object: Literal["chat.completion.chunk"] + choices: list[_BridgeChoice] = Field(default_factory=list) class _WeatherArgs(BaseModel): @@ -103,16 +106,19 @@ class TestResponsesBridgeChatCompletionsStreaming: resources.key(), ChatBody( model=bridged_model, - messages=[ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}")], + messages=[ + ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}") + ], max_tokens=64, stream=True, ), ) - chunks = _bridge_chunks(result) - ids = {chunk.id for chunk in chunks} + chunks: Final = _bridge_chunks(result) + assert len(chunks) > 1, "the shared-id contract needs more than one streamed chunk" + ids: Final = frozenset(chunk.id for chunk in chunks) assert len(ids) == 1, f"bridged stream used {len(ids)} different chunk ids: {sorted(ids)[:5]}" - assert ids.pop().startswith("chatcmpl-"), f"bridged chunk id is not chat-completion shaped: {chunks[0].id}" + assert chunks[0].id.strip(), "bridged stream emitted an empty chunk id" @pytest.mark.covers( "llm.chat_completions.openai.basic.stream.bridge_streams_sse", @@ -134,9 +140,9 @@ class TestResponsesBridgeChatCompletionsStreaming: chunks = _bridge_chunks(result) content = "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) assert content.strip(), f"bridged stream completed with no content deltas: {result.stream_events[:3]}" - assert any( - choice.finish_reason for chunk in chunks for choice in chunk.choices - ), f"bridged stream never emitted a finish_reason: {result.stream_events[-3:]}" + assert any(choice.finish_reason for chunk in chunks for choice in chunk.choices), ( + f"bridged stream never emitted a finish_reason: {result.stream_events[-3:]}" + ) assert result.stream_done, f"bridged stream did not terminate with [DONE]: {result.stream_events[-2:]}" @pytest.mark.covers( diff --git a/tests/e2e/logging/datadog_reader.py b/tests/e2e/logging/datadog_reader.py index d0f478185c2..368c20cb6aa 100644 --- a/tests/e2e/logging/datadog_reader.py +++ b/tests/e2e/logging/datadog_reader.py @@ -12,8 +12,12 @@ empty result. External reads go through ``e2e_http``. from __future__ import annotations +import math +import random import time -from dataclasses import dataclass +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import Final import pytest from pydantic import BaseModel, ConfigDict, Field @@ -27,17 +31,33 @@ from e2e_config import ( DD_SITE, POLL_TIMEOUT, ) -from e2e_http import URL, Headers, RateLimitedError, Success, post +from e2e_http import URL, Headers, StreamingResponse, send -#: How many rate-limited responses in a row one search tolerates before the -#: hard fail; each retry sleeps a full search interval, so this rides out a -#: burst from a concurrent consumer of the org-wide search budget. -_RATE_LIMIT_RETRIES = 5 +type SearchCall = Callable[[str, float], StreamingResponse] + + +def _seconds(value: str | None) -> float | None: + if value is None: + return None + try: + seconds: Final = float(value) + except ValueError: + return None + return seconds if math.isfinite(seconds) and seconds >= 0 else None + + +def _rate_limit_delay(headers: Mapping[str, str]) -> float: + delays: Final = tuple( + delay + for name in ("x-ratelimit-reset", "retry-after") + if (delay := _seconds(headers.get(name))) is not None + ) + return max(1.0, max(delays, default=DD_SEARCH_INTERVAL)) class _DdAuthHeaders(Headers): - api_key: str = Field(serialization_alias="DD-API-KEY") - app_key: str = Field(serialization_alias="DD-APPLICATION-KEY") + api_key: str = Field(serialization_alias="DD-API-KEY", repr=False) + app_key: str = Field(serialization_alias="DD-APPLICATION-KEY", repr=False) class _SearchFilter(BaseModel): @@ -88,8 +108,12 @@ class _SearchResponse(BaseModel): @dataclass(frozen=True, slots=True) class DdLogsReader: site: str - api_key: str - app_key: str + api_key: str = field(repr=False) + app_key: str = field(repr=False) + search: SearchCall | None = field(default=None, repr=False) + now: Callable[[], float] = field(default=time.monotonic, repr=False) + sleep: Callable[[float], None] = field(default=time.sleep, repr=False) + jitter: Callable[[], float] = field(default=random.random, repr=False) def events_for_marker(self, marker: str) -> list[DdLogEvent]: """Every ingested event whose attributes carry the marker. DataDog @@ -108,25 +132,28 @@ class DdLogsReader: a single event. A 429 backs off and retries - the search budget is org-wide, so another consumer can empty it under us - while any other failure stays a hard fail.""" - for _ in range(_RATE_LIMIT_RETRIES): - result = post( - URL(f"https://api.{self.site}/api/v2/logs/events/search"), - headers=_DdAuthHeaders(api_key=self.api_key, app_key=self.app_key), - json=_SearchRequest(filter=_SearchFilter(query=query)), - response_type=_SearchResponse, - timeout=30.0, - ) - match result: - case Success(data=page): - return [event.attributes for event in page.data] - case RateLimitedError(retry_after_seconds=retry_after): - time.sleep(retry_after if retry_after else DD_SEARCH_INTERVAL) - case failure: - pytest.fail(f"DataDog Logs Search API at api.{self.site} failed: {failure}") + return self._events_for_query(query, self.now() + POLL_TIMEOUT) + + def _events_for_query(self, query: str, deadline: float) -> list[DdLogEvent]: + search: Final = self.search or self._search_page + while (remaining := deadline - self.now()) > 0: + if (result := search(query, min(30.0, remaining))).ok: + return [event.attributes for event in _SearchResponse.model_validate_json(result.body).data] + if result.status_code != 429: + pytest.fail(f"DataDog Logs Search API at api.{self.site} failed with HTTP {result.status_code}") + if (delay := min(_rate_limit_delay(result.headers) + self.jitter(), deadline - self.now())) > 0: + self.sleep(delay) pytest.fail( - f"DataDog Logs Search API at api.{self.site} still rate-limited after " - f"{_RATE_LIMIT_RETRIES} retries {DD_SEARCH_INTERVAL}s apart - the org-wide " - "logs_public_search_api budget (2 requests per 10s) is exhausted by another consumer" + f"DataDog Logs Search API at api.{self.site} remained rate-limited for {POLL_TIMEOUT}s; " + "the org-wide logs_public_search_api budget is exhausted" + ) + + def _search_page(self, query: str, timeout: float) -> StreamingResponse: + return send( + URL(f"https://api.{self.site}/api/v2/logs/events/search"), + headers=_DdAuthHeaders(api_key=self.api_key, app_key=self.app_key), + json=_SearchRequest(filter=_SearchFilter(query=query)), + timeout=timeout, ) def poll_events_for_marker(self, marker: str) -> list[DdLogEvent]: @@ -140,33 +167,42 @@ class DdLogsReader: hide from the exactly-one assertion - real-DataDog jitter can surface one call's two events tens of seconds apart. Searches pace at DD_SEARCH_INTERVAL, not POLL_INTERVAL, to respect the search API's - request budget. At the deadline the last result is returned as-is.""" - deadline = time.monotonic() + POLL_TIMEOUT - while time.monotonic() < deadline: - events = self.events_for_query(query) + request budget. Discovery, quota retries, and duplicate detection share + one POLL_TIMEOUT deadline; an incomplete settle window fails closed.""" + deadline: Final = self.now() + POLL_TIMEOUT + while (remaining := deadline - self.now()) > 0: + events = self._events_for_query(query, deadline) if events: - return self._settled_events_for_query(query, events) - time.sleep(DD_SEARCH_INTERVAL) - return self.events_for_query(query) + return self._settled_events_for_query(query, events, deadline) + if (remaining := deadline - self.now()) > 0: + self.sleep(min(DD_SEARCH_INTERVAL, remaining)) + return [] - def _settled_events_for_query(self, query: str, events: list[DdLogEvent]) -> list[DdLogEvent]: + def _settled_events_for_query(self, query: str, events: list[DdLogEvent], deadline: float) -> list[DdLogEvent]: """Re-read at every search interval until the settle window closes; a duplicate ends the watch early because more waiting cannot clear it. Keep the last non-empty result: a transient empty search (index lag) must not erase events already confirmed earlier in the settle window. + A successful final search must reach the full settle window before the + shared read-back deadline; otherwise duplicate detection is incomplete. """ - settle_deadline = time.monotonic() + DD_SETTLE_SECONDS + settle_deadline: Final = self.now() + DD_SETTLE_SECONDS last_nonempty = events - while time.monotonic() < settle_deadline: - time.sleep(DD_SEARCH_INTERVAL) - latest = self.events_for_query(query) - if not latest: - continue + if len(events) > 1: + return events + while (remaining := deadline - self.now()) > 0: + self.sleep(min(DD_SEARCH_INTERVAL, remaining)) + if self.now() >= deadline: + break + latest = self._events_for_query(query, deadline) if len(latest) > 1: return latest - last_nonempty = latest - return last_nonempty + if latest: + last_nonempty = latest + if self.now() >= settle_deadline: + return last_nonempty + pytest.fail(f"DataDog log delivery could not complete its duplicate-detection window within {POLL_TIMEOUT}s") def build_dd_logs_reader() -> DdLogsReader: diff --git a/tests/e2e/logging/test_datadog_reader.py b/tests/e2e/logging/test_datadog_reader.py new file mode 100644 index 00000000000..910a1cefd42 --- /dev/null +++ b/tests/e2e/logging/test_datadog_reader.py @@ -0,0 +1,223 @@ +import json +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from typing import Final + +import pytest + +from datadog_reader import DdLogsReader +from datadog_reader import _DdAuthHeaders # pyright: ignore[reportPrivateUsage] # verifies private auth-header serialization +from e2e_config import DD_SEARCH_INTERVAL, POLL_TIMEOUT +from e2e_http import StreamingResponse + + +def test_failure_diagnostics_hide_credentials_without_changing_auth_headers() -> None: + api_key: Final = "test-datadog-api-secret" + app_key: Final = "test-datadog-app-secret" + reader: Final = DdLogsReader(site="datadoghq.com", api_key=api_key, app_key=app_key) + headers: Final = _DdAuthHeaders(api_key=api_key, app_key=app_key) + + for value in (reader, headers): + assert api_key not in repr(value) + assert app_key not in repr(value) + + assert headers.model_dump(by_alias=True) == { + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + } + + +@dataclass +class Clock: + elapsed: float = 0.0 + + def now(self) -> float: + return self.elapsed + + def sleep(self, seconds: float) -> None: + self.elapsed += seconds + + +@dataclass +class Search: + responses: Iterator[StreamingResponse] + calls: tuple[tuple[str, float], ...] = () + + def __call__(self, query: str, timeout: float) -> StreamingResponse: + self.calls += ((query, timeout),) + return next(self.responses) + + +def _page(*event_ids: str) -> StreamingResponse: + return StreamingResponse( + status_code=200, + body=json.dumps({"data": [{"attributes": {"attributes": {"id": event_id}}} for event_id in event_ids]}), + ) + + +def _reader(responses: Sequence[StreamingResponse], clock: Clock) -> tuple[DdLogsReader, Search]: + search: Final = Search(iter(responses)) + return DdLogsReader( + site="us5.datadoghq.com", + api_key="test-api-secret", + app_key="test-app-secret", + search=search, + now=clock.now, + sleep=clock.sleep, + jitter=lambda: 0.25, + ), search + + +def test_429_honors_server_reset_and_preserves_duplicate_events() -> None: + clock: Final = Clock() + reader, search = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": "6"}), _page("first", "duplicate")), + clock, + ) + + events: Final = reader.events_for_query("test-marker") + + assert tuple(event.attributes["id"] for event in events) == ("first", "duplicate") + assert clock.elapsed == 6.25 + assert search.calls == (("test-marker", 30.0), ("test-marker", 30.0)) + + +@pytest.mark.parametrize("reset", ("", "invalid", "nan", "inf", "-1")) +def test_invalid_reset_uses_search_interval(reset: str) -> None: + clock: Final = Clock() + reader, _ = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": reset}), _page()), clock + ) + + assert reader.events_for_query("test-marker") == [] + assert clock.elapsed == DD_SEARCH_INTERVAL + 0.25 + + +def test_zero_reset_cannot_create_a_busy_retry_loop() -> None: + clock: Final = Clock() + reader, _ = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": "0"}), _page()), clock + ) + + assert reader.events_for_query("test-marker") == [] + assert clock.elapsed == 1.25 + + +def test_retry_after_is_not_shortened_by_an_earlier_reset() -> None: + clock: Final = Clock() + reader, _ = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": "2", "retry-after": "8"}), _page()), + clock, + ) + + assert reader.events_for_query("test-marker") == [] + assert clock.elapsed == 8.25 + + +def test_rate_limit_wait_stops_at_deadline_without_issuing_another_request() -> None: + clock: Final = Clock() + reader, search = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT * 10)}),), clock + ) + + with pytest.raises(pytest.fail.Exception, match="remained rate-limited"): + reader.events_for_query("test-marker") + + assert clock.elapsed == POLL_TIMEOUT + assert search.calls == (("test-marker", 30.0),) + + +def test_late_retry_cannot_receive_a_fresh_request_timeout() -> None: + clock: Final = Clock() + reader, search = _reader( + (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT - 5)}), _page()), + clock, + ) + + assert reader.events_for_query("test-marker") == [] + assert search.calls == (("test-marker", 30.0), ("test-marker", 4.75)) + + +@pytest.mark.parametrize("status", (-1, 401, 403, 500)) +def test_non_quota_failures_are_not_retried_or_treated_as_empty_results(status: int) -> None: + clock: Final = Clock() + reader, search = _reader((StreamingResponse(status_code=status, body=""), _page()), clock) + + with pytest.raises(pytest.fail.Exception, match=f"failed with HTTP {status}"): + reader.events_for_query("test-marker") + + assert search.calls == (("test-marker", 30.0),) + assert clock.elapsed == 0 + + +def test_polling_quota_retries_share_the_original_deadline() -> None: + clock: Final = Clock() + reader, search = _reader( + (_page(), StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT)})), + clock, + ) + + with pytest.raises(pytest.fail.Exception, match="remained rate-limited"): + reader.poll_events_for_query("test-marker") + + assert clock.elapsed == POLL_TIMEOUT + assert len(search.calls) == 2 + + +def test_empty_polling_does_not_start_a_final_search_after_its_deadline() -> None: + clock: Final = Clock() + attempts: Final = int(POLL_TIMEOUT / DD_SEARCH_INTERVAL) + reader, search = _reader((_page(),) * attempts, clock) + + assert reader.poll_events_for_query("test-marker") == [] + assert clock.elapsed == POLL_TIMEOUT + assert len(search.calls) == attempts + + +def test_settlement_quota_retries_keep_the_remaining_readback_budget() -> None: + clock: Final = Clock() + empty_reads: Final = int(POLL_TIMEOUT / DD_SEARCH_INTERVAL) - 2 + reader, search = _reader( + (_page(),) * empty_reads + + (_page("first"), StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT)})), + clock, + ) + + with pytest.raises(pytest.fail.Exception, match="remained rate-limited"): + reader.poll_events_for_query("test-marker") + + assert clock.elapsed == POLL_TIMEOUT + assert search.calls[-1] == ("test-marker", DD_SEARCH_INTERVAL) + assert len(search.calls) == empty_reads + 2 + + +def test_settlement_detects_a_duplicate_on_the_final_search() -> None: + clock: Final = Clock() + reader, _ = _reader((_page("first"), _page("first"), _page(), _page("first", "duplicate")), clock) + + events: Final = reader.poll_events_for_query("test-marker") + + assert tuple(event.attributes["id"] for event in events) == ("first", "duplicate") + assert clock.elapsed == 30 + + +def test_settlement_keeps_confirmed_events_through_empty_searches() -> None: + clock: Final = Clock() + reader, _ = _reader((_page("first"), _page(), _page(), _page()), clock) + + events: Final = reader.poll_events_for_query("test-marker") + + assert tuple(event.attributes["id"] for event in events) == ("first",) + assert clock.elapsed == 30 + + +def test_late_delivery_cannot_pass_without_a_complete_settle_window() -> None: + clock: Final = Clock() + empty_reads: Final = int(POLL_TIMEOUT / DD_SEARCH_INTERVAL) - 2 + reader, search = _reader((_page(),) * empty_reads + (_page("first"), _page("first")), clock) + + with pytest.raises(pytest.fail.Exception, match="duplicate-detection window"): + reader.poll_events_for_query("test-marker") + + assert clock.elapsed == POLL_TIMEOUT + assert len(search.calls) == empty_reads + 2 diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 2b897f5f07f..1ef0d89a8f9 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -43,6 +43,9 @@ from models import ( KeyResetSpendBody, KeyResetSpendResponse, KeyUpdateBody, + McpServerCreateBody, + McpServerRow, + McpServerUpdateBody, ModelDeleteBody, OrgDeleteBody, OrgInfoParams, @@ -537,6 +540,38 @@ class ManagementClient: ).root ) + def create_mcp_server(self, body: McpServerCreateBody) -> McpServerRow: + return unwrap( + self.proxy.transport.post( + "/v1/mcp/server", + headers=self.proxy.transport.master, + json=body, + response_type=McpServerRow, + ) + ) + + def update_mcp_server(self, body: McpServerUpdateBody) -> McpServerRow: + """PUT /v1/mcp/server, the call behind the dashboard's Save Changes: a partial + update where a field left unset keeps its stored value and None clears it.""" + return unwrap( + self.proxy.transport.put( + "/v1/mcp/server", + headers=self.proxy.transport.master, + json=body, + response_type=McpServerRow, + ) + ) + + def delete_mcp_server(self, server_id: str) -> Result[NoBody]: + """DELETE /v1/mcp/server/{server_id}. Returns the outcome so the act phase can + unwrap it while a deferred teardown can ignore an already-deleted server.""" + return self.proxy.transport.delete( + f"/v1/mcp/server/{server_id}", + headers=self.proxy.transport.master, + json=NoBody(), + response_type=NoBody, + ) + def chat_status(self, key: str, model: str, content: str) -> StreamingResponse: return self.proxy.transport.send( "/chat/completions", diff --git a/tests/e2e/management/test_mcp_lifecycle_e2e.py b/tests/e2e/management/test_mcp_lifecycle_e2e.py new file mode 100644 index 00000000000..9257d697647 --- /dev/null +++ b/tests/e2e/management/test_mcp_lifecycle_e2e.py @@ -0,0 +1,294 @@ +"""Live e2e: the MCP server and toolset management routes' lifecycle contract. + +Two customer defects sit on these routes, and each step here is the read-back that +would have caught one of them: a dashboard edit that took several saves to stick +because the read landed on a replica the write had not reached, and a toolset whose +tools were stored under one name and read back under another, so it granted +nothing. Every read-back therefore polls every replica that serves the route +(ProxyClient.read_back_everywhere) and asserts the exact values written, and both +update routes are held to the same partial-update contract: a field left out of the +payload keeps its stored value, a field sent as null is cleared. The server URL is +unreachable on purpose; only persistence is under test, never a tool call. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import ( + McpInfo, + McpServerCreateBody, + McpServerListResponse, + McpServerRow, + McpServerUpdateBody, + ToolsetCreateBody, + ToolsetListResponse, + ToolsetRow, + ToolsetTool, + ToolsetUpdateBody, +) + +pytestmark = pytest.mark.e2e + +UNREACHABLE_URL: Final = "https://e2e-fake-mcp.test.local/mcp" + + +def _create_server(client: ManagementClient, resources: ResourceManager) -> tuple[McpServerCreateBody, str]: + name: Final = f"e2e_mcp_lifecycle_{unique_marker()}" + body: Final = McpServerCreateBody( + server_name=name, + alias=name, + url=UNREACHABLE_URL, + transport="http", + description="e2e lifecycle server", + mcp_info=McpInfo( + server_name=f"{name} (display)", + description="shown on the MCP page", + logo_url="https://e2e.test.local/logo.png", + ), + ) + server_id: Final = client.create_mcp_server(body).server_id + resources.defer(lambda: client.delete_mcp_server(server_id)) + return body, server_id + + +def _assert_server_matches(row: McpServerRow, written: McpServerCreateBody, *, where: str) -> None: + stored: Final = (row.server_name, row.alias, row.url, row.transport, row.description, row.mcp_info) + expected: Final = ( + written.server_name, + written.alias, + written.url, + written.transport, + written.description, + written.mcp_info, + ) + assert stored == expected, f"{where}: stored {stored}, expected {expected}" + + +def _server_everywhere( + client: ManagementClient, server_id: str, *, settled: Callable[[McpServerRow], bool] +) -> Mapping[str, McpServerRow]: + return client.proxy.read_body_back_everywhere(f"/v1/mcp/server/{server_id}", McpServerRow, settled=settled) + + +def _listed_server_everywhere(client: ManagementClient, server_id: str) -> Mapping[str, McpServerRow]: + listings: Final = client.proxy.read_body_back_everywhere( + "/v1/mcp/server", + McpServerListResponse, + settled=lambda rows: any(row.server_id == server_id for row in rows.root), + ) + return {replica: next(row for row in rows.root if row.server_id == server_id) for replica, rows in listings.items()} + + +class TestMcpServerLifecycle: + @pytest.mark.covers("mgmt.mcp_server.new.persists") + def test_create_persists_every_field_on_every_replica( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + body, server_id = _create_server(client, resources) + + by_id: Final = _server_everywhere(client, server_id, settled=lambda row: row.server_id == server_id) + for replica, row in by_id.items(): + _assert_server_matches(row, body, where=f"GET /v1/mcp/server/{server_id} on {replica}") + + @pytest.mark.skip( + reason=( + "product gap: GET /v1/mcp/server builds each row from the in-memory registry, whose " + "_build_mcp_server_table sets description from mcp_info['description'], so the list " + "reports the mcp_info description while GET /v1/mcp/server/{server_id} reports the " + "stored description column. A server created with both set to different text reads " + "back with two different descriptions depending on the route" + ) + ) + @pytest.mark.covers("mgmt.mcp_server.list.persists") + def test_created_server_is_listed_with_every_field( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + body, server_id = _create_server(client, resources) + + for replica, row in _listed_server_everywhere(client, server_id).items(): + _assert_server_matches(row, body, where=f"GET /v1/mcp/server on {replica}") + + @pytest.mark.covers("mgmt.mcp_server.update.preserves_unrelated_fields") + def test_updating_only_the_alias_keeps_every_other_field_on_every_replica( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + body, server_id = _create_server(client, resources) + renamed: Final = f"{body.alias}_renamed" + + _ = client.update_mcp_server(McpServerUpdateBody(server_id=server_id, alias=renamed)) + + after_one_put: Final = _server_everywhere(client, server_id, settled=lambda row: row.alias == renamed) + for replica, row in after_one_put.items(): + _assert_server_matches( + row, + body.model_copy(update={"alias": renamed}), + where=f"GET /v1/mcp/server/{server_id} on {replica} after one PUT of alias", + ) + + @pytest.mark.covers("mgmt.mcp_server.update.clear_persists") + def test_clearing_the_description_with_null_reads_back_null( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + body, server_id = _create_server(client, resources) + + _ = client.update_mcp_server(McpServerUpdateBody(server_id=server_id, description=None)) + + cleared: Final = _server_everywhere(client, server_id, settled=lambda row: row.description is None) + for replica, row in cleared.items(): + _assert_server_matches( + row, + body.model_copy(update={"description": None}), + where=f"GET /v1/mcp/server/{server_id} on {replica} after PUT description=null", + ) + + @pytest.mark.covers("mgmt.mcp_server.delete.persists") + def test_delete_removes_the_server_from_every_replica( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + _, server_id = _create_server(client, resources) + + _ = unwrap(client.delete_mcp_server(server_id)) + + gone: Final = client.proxy.gone_everywhere(f"/v1/mcp/server/{server_id}") + assert set(gone.values()) == {404}, f"a deleted server must 404 on every replica; got {dict(gone)}" + listings: Final = client.proxy.read_body_back_everywhere( + "/v1/mcp/server", + McpServerListResponse, + settled=lambda rows: all(row.server_id != server_id for row in rows.root), + ) + for replica, rows in listings.items(): + assert all(row.server_id != server_id for row in rows.root), ( + f"GET /v1/mcp/server on {replica} still lists the deleted server {server_id}" + ) + + +def _create_toolset( + client: ManagementClient, resources: ResourceManager, server_id: str +) -> tuple[ToolsetCreateBody, str]: + body: Final = ToolsetCreateBody( + toolset_name=f"e2e_toolset_{unique_marker()}", + description="e2e lifecycle toolset", + tools=[ + ToolsetTool(server_id=server_id, tool_name="search_datadog_logs"), + ToolsetTool(server_id=server_id, tool_name="get_datadog_metric"), + ], + ) + toolset_id: Final = client.proxy.create_toolset(body).toolset_id + resources.defer(lambda: client.proxy.delete_toolset(toolset_id)) + return body, toolset_id + + +def _assert_toolset_matches(row: ToolsetRow, written: ToolsetCreateBody, *, where: str) -> None: + stored: Final = (row.toolset_name, row.description, row.tools) + expected: Final = (written.toolset_name, written.description, written.tools) + assert stored == expected, f"{where}: stored {stored}, expected {expected}" + + +def _toolset_everywhere( + client: ManagementClient, toolset_id: str, *, settled: Callable[[ToolsetRow], bool] +) -> Mapping[str, ToolsetRow]: + return client.proxy.read_body_back_everywhere(f"/v1/mcp/toolset/{toolset_id}", ToolsetRow, settled=settled) + + +class TestMcpToolsetLifecycle: + @pytest.mark.covers("mgmt.mcp_toolset.new.persists") + def test_create_persists_both_tools_under_the_exact_names_written( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + _, server_id = _create_server(client, resources) + body, toolset_id = _create_toolset(client, resources, server_id) + + by_id: Final = _toolset_everywhere(client, toolset_id, settled=lambda row: row.toolset_id == toolset_id) + for replica, row in by_id.items(): + _assert_toolset_matches(row, body, where=f"GET /v1/mcp/toolset/{toolset_id} on {replica}") + listings: Final = client.proxy.read_body_back_everywhere( + "/v1/mcp/toolset", + ToolsetListResponse, + settled=lambda rows: any(row.toolset_id == toolset_id for row in rows.root), + ) + for replica, rows in listings.items(): + _assert_toolset_matches( + next(row for row in rows.root if row.toolset_id == toolset_id), + body, + where=f"GET /v1/mcp/toolset on {replica}", + ) + + @pytest.mark.covers("mgmt.mcp_toolset.update.preserves_unrelated_fields") + def test_updating_only_the_description_keeps_the_tools_and_name( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + _, server_id = _create_server(client, resources) + body, toolset_id = _create_toolset(client, resources, server_id) + + _ = client.proxy.update_toolset(ToolsetUpdateBody(toolset_id=toolset_id, description="edited")) + + edited: Final = _toolset_everywhere(client, toolset_id, settled=lambda row: row.description == "edited") + for replica, row in edited.items(): + _assert_toolset_matches( + row, + body.model_copy(update={"description": "edited"}), + where=f"GET /v1/mcp/toolset/{toolset_id} on {replica} after PUT of description", + ) + + @pytest.mark.covers("mgmt.mcp_toolset.update.persists") + def test_updating_the_tools_to_one_entry_reads_back_exactly_that_entry( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + _, server_id = _create_server(client, resources) + body, toolset_id = _create_toolset(client, resources, server_id) + kept: Final = body.tools[:1] + + _ = client.proxy.update_toolset(ToolsetUpdateBody(toolset_id=toolset_id, tools=kept)) + + narrowed: Final = _toolset_everywhere(client, toolset_id, settled=lambda row: row.tools == kept) + for replica, row in narrowed.items(): + _assert_toolset_matches( + row, + body.model_copy(update={"tools": kept}), + where=f"GET /v1/mcp/toolset/{toolset_id} on {replica} after PUT of one tool", + ) + + @pytest.mark.covers("mgmt.mcp_toolset.update.clear_persists") + def test_clearing_the_description_with_null_reads_back_null( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + _, server_id = _create_server(client, resources) + body, toolset_id = _create_toolset(client, resources, server_id) + + _ = client.proxy.update_toolset(ToolsetUpdateBody(toolset_id=toolset_id, description=None)) + + cleared: Final = _toolset_everywhere(client, toolset_id, settled=lambda row: row.description is None) + for replica, row in cleared.items(): + _assert_toolset_matches( + row, + body.model_copy(update={"description": None}), + where=f"GET /v1/mcp/toolset/{toolset_id} on {replica} after PUT description=null", + ) + + @pytest.mark.covers("mgmt.mcp_toolset.delete.persists") + def test_delete_removes_the_toolset_from_every_replica( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + _, server_id = _create_server(client, resources) + _, toolset_id = _create_toolset(client, resources, server_id) + + _ = unwrap(client.proxy.delete_toolset(toolset_id)) + + gone: Final = client.proxy.gone_everywhere(f"/v1/mcp/toolset/{toolset_id}") + assert set(gone.values()) == {404}, f"a deleted toolset must 404 on every replica; got {dict(gone)}" + listings: Final = client.proxy.read_body_back_everywhere( + "/v1/mcp/toolset", + ToolsetListResponse, + settled=lambda rows: all(row.toolset_id != toolset_id for row in rows.root), + ) + for replica, rows in listings.items(): + assert all(row.toolset_id != toolset_id for row in rows.root), ( + f"GET /v1/mcp/toolset on {replica} still lists the deleted toolset {toolset_id}" + ) diff --git a/tests/e2e/mcp/datadog_mcp.py b/tests/e2e/mcp/datadog_mcp.py index d1ea53a0b3b..352b4446cfd 100644 --- a/tests/e2e/mcp/datadog_mcp.py +++ b/tests/e2e/mcp/datadog_mcp.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from collections.abc import Sequence from e2e_config import datadog_mcp_url, unique_marker from lifecycle import ResourceManager @@ -35,7 +36,11 @@ def register_datadog_mcp( resources: ResourceManager, *, mcp_access_groups: list[str] | None = None, + allowed_tools: Sequence[str] | None = (SEARCH_LOGS_TOOL,), ) -> str: + """Register the core Datadog toolset with its credentials from the env. By default + the server exposes only `search_datadog_logs`; pass `allowed_tools=None` to expose + every tool the core toolset serves.""" assert_dd_mcp_creds() name = f"e2e_dd_mcp_{unique_marker()}" server_id = client.register_server( @@ -47,7 +52,7 @@ def register_datadog_mcp( "DD-API-KEY": _dd_api_key(), "DD-APPLICATION-KEY": _dd_app_key(), }, - allowed_tools=[SEARCH_LOGS_TOOL], + allowed_tools=None if allowed_tools is None else list(allowed_tools), mcp_access_groups=mcp_access_groups, ) resources.defer(lambda: client.delete_server(server_id)) diff --git a/tests/e2e/mcp/mcp_client.py b/tests/e2e/mcp/mcp_client.py index 73453478e5a..210fc7a1e98 100644 --- a/tests/e2e/mcp/mcp_client.py +++ b/tests/e2e/mcp/mcp_client.py @@ -16,11 +16,11 @@ import time from collections.abc import Mapping from dataclasses import dataclass -from pydantic import BaseModel, ConfigDict, Field, RootModel +from pydantic import BaseModel, ConfigDict, Field from e2e_config import settle_propagation from e2e_http import Headers, NoBody, Result, Success, UnknownApiError, unwrap -from models import KeyGenerateBody, ObjectPermission +from models import KeyGenerateBody, McpServerListResponse, McpServerRow, ObjectPermission from proxy_client import ProxyClient McpToolArg = str | int | float | bool | list[str] | dict[str, str] @@ -46,16 +46,6 @@ class McpServerNewResponse(BaseModel): server_id: str -class McpServerRow(BaseModel): - server_id: str - alias: str | None = None - url: str | None = None - - -class McpServersListResponse(RootModel[list[McpServerRow]]): - pass - - class McpToolMcpInfo(BaseModel): server_id: str | None = None alias: str | None = None @@ -193,7 +183,7 @@ class McpClient: "/v1/mcp/server", headers=self.proxy.transport.master, params=NoBody(), - response_type=McpServersListResponse, + response_type=McpServerListResponse, ) ).root @@ -224,11 +214,16 @@ class McpClient: user_id: str, mcp_servers: list[str] | None, mcp_access_groups: list[str] | None = None, + mcp_toolsets: list[str] | None = None, models: list[str] | None = None, ) -> str: object_permission = ( - ObjectPermission(mcp_servers=mcp_servers, mcp_access_groups=mcp_access_groups) - if mcp_servers is not None or mcp_access_groups is not None + ObjectPermission( + mcp_servers=mcp_servers, + mcp_access_groups=mcp_access_groups, + mcp_toolsets=mcp_toolsets, + ) + if mcp_servers is not None or mcp_access_groups is not None or mcp_toolsets is not None else None ) return self.proxy.generate_key( @@ -272,6 +267,20 @@ class McpClient: ) time.sleep(self.proxy.poll_interval) + def await_tools(self, key: str, server_id: str, *, expected: frozenset[str]) -> frozenset[str]: + """Poll tools/list until `server_id`'s tools as `key` sees them are exactly + `expected`, and return the last listing either way, so the caller's equality + assertion names the difference. Fails at poll_timeout only when the read + itself never succeeded.""" + deadline = time.monotonic() + self.proxy.poll_timeout + while True: + result = self.list_tools(key) + if isinstance(result, Success) and result.data.tool_names_for_server(server_id) == expected: + return expected + if time.monotonic() >= deadline: + return unwrap(result).tool_names_for_server(server_id) + time.sleep(self.proxy.poll_interval) + def await_call_tool( self, key: str, diff --git a/tests/e2e/mcp/test_mcp_toolset_enforcement_e2e.py b/tests/e2e/mcp/test_mcp_toolset_enforcement_e2e.py new file mode 100644 index 00000000000..6b901145eb1 --- /dev/null +++ b/tests/e2e/mcp/test_mcp_toolset_enforcement_e2e.py @@ -0,0 +1,95 @@ +"""Live e2e: a key granted a toolset lists exactly the toolset's tools. + +An admin registers the real Datadog remote MCP server with its whole core toolset +exposed, discovers two of its tool names through a key granted the server outright, +and curates a toolset naming exactly those two. A second key is granted the server +plus that toolset, and its tools/list must come back as exactly those two names: no +more, so the rest of the server's catalog stays hidden behind the toolset, and no +fewer, so a tool stored under one name and read under another (which granted +nothing) fails here first. Requires DD_API_KEY + DD_APP_KEY (the suite's real MCP +upstream). +""" + +from __future__ import annotations + +from typing import Final + +import pytest +from datadog_mcp import SEARCH_LOGS_TOOL, register_datadog_mcp +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from mcp_client import McpClient +from models import ToolsetCreateBody, ToolsetTool + +pytestmark = pytest.mark.e2e + + +def _key( + client: McpClient, + resources: ResourceManager, + label: str, + *, + server_id: str, + toolset_id: str | None = None, +) -> str: + key: Final = client.generate_key( + user_id=f"e2e-mcp-{label}-{unique_marker()}", + mcp_servers=[server_id], + mcp_toolsets=None if toolset_id is None else [toolset_id], + ) + resources.defer(lambda: client.proxy.delete_key(key)) + return key + + +def _wire_prefix(wire_name: str, tool_name: str, catalog: frozenset[str]) -> str: + """The prefix tools/list puts in front of one server's tool names, measured off a + tool whose own name is known rather than guessed from the alias. A toolset grants + by the tool's own name, never the wire name, and the prefix is whatever the proxy + is configured to build (the alias, or a short server id), so measuring it is the + only way to cross between the two.""" + assert wire_name.endswith(tool_name), f"tools/list served {wire_name!r}, expected it to end with {tool_name!r}" + prefix: Final = wire_name[: len(wire_name) - len(tool_name)] + unprefixed: Final = frozenset(name for name in catalog if not name.startswith(prefix)) + assert not unprefixed, ( + f"every tool of one server shares the wire prefix {prefix!r}, so {sorted(unprefixed)} " + f"cannot be reduced to the names a toolset grants by" + ) + return prefix + + +class TestMcpToolsetEnforcement: + @pytest.mark.covers("mcp.list_tools.api_key.toolset_scoped") + def test_key_granted_a_toolset_lists_exactly_its_tools(self, client: McpClient, resources: ResourceManager) -> None: + server_id: Final = register_datadog_mcp(client, resources, allowed_tools=None) + client.await_registered(server_id) + + catalog_key: Final = _key(client, resources, "catalog", server_id=server_id) + known_wire: Final = client.await_tool(catalog_key, server_id, SEARCH_LOGS_TOOL) + catalog: Final = unwrap(client.list_tools(catalog_key)).tool_names_for_server(server_id) + assert len(catalog) > 2, ( + f"the Datadog core toolset must serve more tools than the toolset names, or the " + f"restriction has nothing to hide; got {sorted(catalog)}" + ) + prefix: Final = _wire_prefix(known_wire, SEARCH_LOGS_TOOL, catalog) + chosen_wire: Final = frozenset(sorted(catalog)[:2]) + chosen: Final = frozenset(name.removeprefix(prefix) for name in chosen_wire) + + toolset: Final = client.proxy.create_toolset( + ToolsetCreateBody( + toolset_name=f"e2e_toolset_{unique_marker()}", + description="two Datadog tools", + tools=[ToolsetTool(server_id=server_id, tool_name=name) for name in sorted(chosen)], + ) + ) + resources.defer(lambda: client.proxy.delete_toolset(toolset.toolset_id)) + assert frozenset(tool.tool_name for tool in toolset.tools) == chosen, ( + f"toolset stored {toolset.tools}, expected the two names {sorted(chosen)} verbatim" + ) + + scoped_key: Final = _key(client, resources, "toolset", server_id=server_id, toolset_id=toolset.toolset_id) + listed: Final = client.await_tools(scoped_key, server_id, expected=chosen_wire) + assert listed == chosen_wire, ( + f"a key granted the toolset must list exactly its two tools; " + f"got {sorted(listed)}, expected {sorted(chosen_wire)}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 9f2654e0eec..62810e6cfd9 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -10,6 +10,7 @@ from collections.abc import Sequence from datetime import datetime from typing import Final, Literal +from e2e_http import PartialBody from pydantic import AliasChoices, BaseModel, ConfigDict, Field, RootModel, model_serializer, model_validator # ---------- keys ---------- @@ -55,6 +56,7 @@ class KeyMetadata(BaseModel): class ObjectPermission(BaseModel): mcp_servers: list[str] | None = None mcp_access_groups: list[str] | None = None + mcp_toolsets: list[str] | None = None class KeyGenerateBody(BaseModel): @@ -77,7 +79,7 @@ class KeyGenerateBody(BaseModel): allowed_passthrough_routes: list[str] | None = None metadata: KeyMetadata | None = None object_permission: ObjectPermission | None = None - router_settings: "RouterSettingsOverride | None" = None + router_settings: RouterSettingsOverride | None = None class KeyGenerateResponse(BaseModel): @@ -516,6 +518,15 @@ class CountTokensResponse(BaseModel): # ---------- mcp servers ---------- +class McpInfo(BaseModel): + """The `mcp_info` display block stored on an MCP server; only the fields the + lifecycle test writes and reads back.""" + + server_name: str | None = None + description: str | None = None + logo_url: str | None = None + + class McpServerCreateBody(BaseModel): """POST /v1/mcp/server. For a gateway-managed OAuth server, `auth_type` is `oauth2` and `oauth2_flow` is `authorization_code`; the upstream endpoints @@ -530,6 +541,18 @@ class McpServerCreateBody(BaseModel): oauth2_flow: Literal["client_credentials", "authorization_code"] | None = None authorization_url: str | None = None token_url: str | None = None + server_name: str | None = None + description: str | None = None + mcp_info: McpInfo | None = None + + +class McpServerUpdateBody(PartialBody): + """PUT /v1/mcp/server: a field left unset keeps its stored value, a field set + to None is cleared.""" + + server_id: str + alias: str | None = None + description: str | None = None class McpServerInfo(BaseModel): @@ -543,6 +566,54 @@ class McpServerInfo(BaseModel): allow_all_keys: bool | None = None +class McpServerRow(McpServerInfo): + """A stored MCP server as the create, get, and list routes return it: the + fields the lifecycle test asserts survive the round trip.""" + + server_name: str | None = None + transport: str | None = None + description: str | None = None + mcp_info: McpInfo | None = None + + +class McpServerListResponse(RootModel[list[McpServerRow]]): + """GET /v1/mcp/server answers with a bare array of servers.""" + + +class ToolsetTool(BaseModel): + server_id: str + tool_name: str + + +class ToolsetCreateBody(BaseModel): + toolset_name: str + description: str | None = None + tools: list[ToolsetTool] + + +class ToolsetUpdateBody(PartialBody): + """PUT /v1/mcp/toolset: a field left unset keeps its stored value, a field set + to None is cleared.""" + + toolset_id: str + description: str | None = None + tools: list[ToolsetTool] | None = None + + +class ToolsetRow(BaseModel): + """A stored toolset as POST /v1/mcp/toolset, GET /v1/mcp/toolset/{toolset_id}, + and each row of GET /v1/mcp/toolset return it.""" + + toolset_id: str + toolset_name: str + description: str | None = None + tools: list[ToolsetTool] = Field(default_factory=list) + + +class ToolsetListResponse(RootModel[list[ToolsetRow]]): + """GET /v1/mcp/toolset answers with a bare array of toolsets.""" + + class EmbedBody(BaseModel): model: str input: str diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 520cbfde5a9..1bac5116a9d 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -12,6 +12,7 @@ import time import warnings from collections.abc import Callable, Mapping from dataclasses import dataclass +from functools import reduce from datetime import datetime from types import MappingProxyType from typing import Final @@ -26,6 +27,7 @@ from e2e_http import ( Result, StreamingResponse, Success, + UnknownApiError, is_ok, unwrap, ) @@ -70,6 +72,9 @@ from models import ( SpendLogsPage, SpendLogsPageParams, SpendLogsParams, + ToolsetCreateBody, + ToolsetRow, + ToolsetUpdateBody, ) from e2e_config import ( CONTROL_PLANE_BASE_URL, @@ -82,7 +87,7 @@ from e2e_config import ( SLOW_PROVIDER_TIMEOUT_SECONDS, settle_propagation, ) -from transport import HttpTransport, SplitTransport, Transport +from transport import HttpTransport, SplitTransport, Transport, is_control_plane_path RowsPredicate = Callable[[list[SpendLogRow]], bool] @@ -235,6 +240,99 @@ def servable_timeout_message( ) +type ReplicaRead[T] = Callable[[float], T] + + +@dataclass(frozen=True, slots=True) +class EverywhereConverged[T]: + """Every replica answered with something `settled` accepts, keyed by replica.""" + + answers: Mapping[str, T] + + +@dataclass(frozen=True, slots=True) +class NeverConvergedOn[T]: + """`replica` ran out its budget without an answer `settled` accepts; `last` is + its final answer, so the failure can say what that replica still serves.""" + + replica: str + last: T + + +def _last_answer[T]( + read: ReplicaRead[T], + *, + settled: Callable[[T], bool], + timeout: float, + interval: float, + request_timeout: float, + now: Callable[[], float], + sleep: Callable[[float], None], +) -> T: + """Poll `read` until `settled` accepts its answer or `timeout` runs out, and + return the last answer either way. Each read's request timeout is clamped to + the budget left, and the final poll runs even when less than an interval + remains, so a deadline never skips the read that would have settled.""" + deadline: Final = now() + timeout + answer = read(min(request_timeout, timeout)) + while not settled(answer): + remaining = deadline - now() + if remaining <= 0: + return answer + sleep(min(interval, remaining)) + answer = read(min(request_timeout, remaining)) + return answer + + +def await_everywhere[T]( + reads: Mapping[str, ReplicaRead[T]], + *, + settled: Callable[[T], bool], + timeout: float, + interval: float, + request_timeout: float, + now: Callable[[], float], + sleep: Callable[[float], None], +) -> EverywhereConverged[T] | NeverConvergedOn[T]: + """`_last_answer` against every replica in turn, each with the full budget, so a + write counts as visible only once the last replica reflects it, and stop at the + first replica that never converges. Clock and sleep are injected.""" + def read_replica( + outcome: EverywhereConverged[T] | NeverConvergedOn[T], + item: tuple[str, ReplicaRead[T]], + ) -> EverywhereConverged[T] | NeverConvergedOn[T]: + if isinstance(outcome, NeverConvergedOn): + return outcome + replica, read = item + answer: Final = _last_answer( + read, + settled=settled, + timeout=timeout, + interval=interval, + request_timeout=request_timeout, + now=now, + sleep=sleep, + ) + if not settled(answer): + return NeverConvergedOn(replica=replica, last=answer) + return EverywhereConverged(answers=MappingProxyType({**outcome.answers, replica: answer})) + + initial: Final[EverywhereConverged[T] | NeverConvergedOn[T]] = EverywhereConverged(answers=MappingProxyType({})) + return reduce(read_replica, reads.items(), initial) + + +def _is_not_found[R: BaseModel](result: Result[R]) -> bool: + return isinstance(result, UnknownApiError) and result.status_code == 404 + + +def _status_of[R: BaseModel](result: Result[R]) -> int: + match result: + case Success(status_code=status_code) | UnknownApiError(status_code=status_code): + return status_code + case _: + return -1 + + type Poller[T] = Callable[[], T] @@ -321,6 +419,7 @@ def converge_timeout_message(*, what: str, replica: str, timeout: float, last_re class ProxyClient: transport: Transport replicas: Mapping[str, Transport] + control_replicas: Mapping[str, Transport] poll_timeout: float = 120.0 poll_interval: float = 5.0 model_servable_timeout: float = MODEL_SERVABLE_TIMEOUT @@ -569,6 +668,112 @@ class ProxyClient: if not is_ok(result): warnings.warn(f"delete_model({model_id!r}) failed: {result}", stacklevel=2) + # ---- replica read-back ---------------------------------------------- + + def replicas_for(self, path: str) -> Mapping[str, Transport]: + """The replicas that serve `path`: every data-plane replica for an LLM route, + and for a management route the control-plane replicas, since the data-plane + replicas trim management routes and answer them 404. A monolith serves both + from every replica, so a management read-back polls all of them; a split + deployment exposes one control-plane address (there is one backend process + behind it on the stack these suites run against), so it polls that. A + control plane fronting several backends would need its own replica list to + prove each one converged, the way PROXY_REPLICA_URLS does for the gateways. + Never empty: a read-back against no replica would assert nothing and pass.""" + replicas: Final = self.control_replicas if is_control_plane_path(path) else self.replicas + assert replicas, f"no replica is configured to serve {path}, so a read-back there would prove nothing" + return replicas + + def read_body_back_everywhere[R: BaseModel]( + self, path: str, response_type: type[R], *, settled: Callable[[R], bool] + ) -> Mapping[str, R]: + """GET `path` on every replica that serves it, polling each to poll_timeout + until `settled` accepts its body, and fail naming the first replica that + never converged. Returns each replica's settled body, keyed by replica, so + the caller can assert the rest of it.""" + outcome: Final = await_everywhere( + {url: self._reader(transport, path, response_type) for url, transport in self.replicas_for(path).items()}, + settled=lambda result: isinstance(result, Success) and settled(result.data), + timeout=self.poll_timeout, + interval=self.poll_interval, + request_timeout=REQUEST_TIMEOUT, + now=time.monotonic, + sleep=time.sleep, + ) + match outcome: + case EverywhereConverged(answers=answers): + return MappingProxyType({url: unwrap(result) for url, result in answers.items()}) + case NeverConvergedOn(replica=replica, last=last): + raise AssertionError( + f"GET {path} on {replica} never converged within {self.poll_timeout}s of the write; " + f"last read: {last}" + ) + + def gone_everywhere(self, path: str) -> Mapping[str, int]: + """Poll GET `path` on every replica that serves it until each stops serving + it, and fail naming the first replica that still does at poll_timeout. + Returns each replica's final status, so the caller asserts the 404 itself.""" + outcome: Final = await_everywhere( + {url: self._reader(transport, path, NoBody) for url, transport in self.replicas_for(path).items()}, + settled=_is_not_found, + timeout=self.poll_timeout, + interval=self.poll_interval, + request_timeout=REQUEST_TIMEOUT, + now=time.monotonic, + sleep=time.sleep, + ) + match outcome: + case EverywhereConverged(answers=answers): + return MappingProxyType({url: _status_of(result) for url, result in answers.items()}) + case NeverConvergedOn(replica=replica, last=last): + raise AssertionError( + f"GET {path} on {replica} still answers {self.poll_timeout}s after the delete; last read: {last}" + ) + + @staticmethod + def _reader[R: BaseModel](transport: Transport, path: str, response_type: type[R]) -> ReplicaRead[Result[R]]: + return lambda request_timeout: transport.get( + path, + headers=transport.master, + params=NoBody(), + response_type=response_type, + timeout=request_timeout, + ) + + # ---- mcp toolsets --------------------------------------------------- + + def create_toolset(self, body: ToolsetCreateBody) -> ToolsetRow: + return unwrap( + self.transport.post( + "/v1/mcp/toolset", + headers=self.transport.master, + json=body, + response_type=ToolsetRow, + ) + ) + + def update_toolset(self, body: ToolsetUpdateBody) -> ToolsetRow: + """PUT /v1/mcp/toolset: a partial update where a field left unset keeps its + stored value and None clears it.""" + return unwrap( + self.transport.put( + "/v1/mcp/toolset", + headers=self.transport.master, + json=body, + response_type=ToolsetRow, + ) + ) + + def delete_toolset(self, toolset_id: str) -> Result[NoBody]: + """DELETE /v1/mcp/toolset/{toolset_id}. Returns the outcome so the act phase + can unwrap it while a deferred teardown can ignore an already-deleted row.""" + return self.transport.delete( + f"/v1/mcp/toolset/{toolset_id}", + headers=self.transport.master, + json=NoBody(), + response_type=NoBody, + ) + def create_credential(self, body: CredentialCreateBody) -> None: unwrap( self.transport.post( @@ -736,7 +941,10 @@ def build_proxy_client( base URLs are the same for a monolithic proxy, so routing is then a no-op. ``replica_urls`` (PROXY_REPLICA_URLS) names every data-plane replica the model barrier polls directly; it is the data-plane URL itself unless the stack - exports each gateway's own address. + exports each gateway's own address. Management read-backs poll those same + replicas when the two planes share a base URL (a monolith, where every replica + serves every route) and the control plane alone when they differ (a split + deployment, where the data-plane replicas do not serve management routes). The endpoints are injectable for callers that resolve the proxy some other way than ``e2e_config``'s env names (see ``claude_code/_env.py``); they must @@ -764,9 +972,13 @@ def build_proxy_client( for url in replica_urls } ) + control_replicas: Final = ( + replicas if control_plane_base_url == base_url else MappingProxyType({control_plane_base_url: split.control}) + ) return ProxyClient( transport=split, replicas=replicas, + control_replicas=control_replicas, poll_timeout=POLL_TIMEOUT, poll_interval=POLL_INTERVAL, ) diff --git a/tests/e2e/router/test_auto_router_regressions_e2e.py b/tests/e2e/router/test_auto_router_regressions_e2e.py index 188db2a8eb5..374badcf5fc 100644 --- a/tests/e2e/router/test_auto_router_regressions_e2e.py +++ b/tests/e2e/router/test_auto_router_regressions_e2e.py @@ -41,6 +41,7 @@ which stores either the registered alias or the provider-prefixed form. import json import os from collections.abc import Iterator +from contextlib import ExitStack from dataclasses import dataclass from typing import Final @@ -120,19 +121,10 @@ class ResponsesApiResponse(BaseModel): @dataclass(frozen=True, slots=True) -class TagSplitDeployments: - """Scenario A mirrors the customer-shaped config from GitHub issue #36619: - plain deployment registered first, tier deployment and marker both tagged. - Scenario B flips both axes for GitHub issue #36621: marker registered first - and its tier deployment left untagged, so routing depends neither on - registration order nor on tier deployments carrying tags.""" - - tag_a: str - shared_a: str - tier_a: str - tag_b: str - shared_b: str - tier_b: str +class TagSplitDeployment: + tag: str + shared: str + tier: str @dataclass(frozen=True, slots=True) @@ -173,9 +165,7 @@ def _uniform_tier_config(tier_model: str) -> dict[str, object]: } -def _key_for( - proxy: ProxyClient, resources: ResourceManager, models: list[str], tag_filtering: bool = False -) -> str: +def _key_for(proxy: ProxyClient, resources: ResourceManager, models: list[str], tag_filtering: bool = False) -> str: key: Final = proxy.generate_key( KeyGenerateBody( models=models, @@ -211,46 +201,61 @@ def _assert_served_only_by(rows: list[SpendLogRow], allowed: frozenset[str], con ) -@pytest.fixture(scope="module") -def split(proxy: ProxyClient) -> Iterator[TagSplitDeployments]: +@pytest.fixture(scope="class") +def router_stack() -> Iterator[ExitStack]: + with ExitStack() as stack: + yield stack + + +def _register_models( + proxy: ProxyClient, stack: ExitStack, registrations: tuple[tuple[str, LiteLLMParamsBody], ...] +) -> None: + for name, params in registrations: + stack.callback(proxy.delete_model, proxy.create_model(name, params)) + + +def _tag_split(proxy: ProxyClient, stack: ExitStack, *, marker_first: bool) -> TagSplitDeployment: marker: Final = unique_marker() - deployments: Final = TagSplitDeployments( - tag_a=f"e2e-split-a-{marker}", - shared_a=f"e2e-autoroute-a-{marker}", - tier_a=f"e2e-tier-a-{marker}", - tag_b=f"e2e-split-b-{marker}", - shared_b=f"e2e-autoroute-b-{marker}", - tier_b=f"e2e-tier-b-{marker}", + named: Final = TagSplitDeployment( + tag=f"e2e-split-{marker}", + shared=f"e2e-autoroute-{marker}", + tier=f"e2e-tier-{marker}", ) anthropic_key: Final = _provider_key("ANTHROPIC_API_KEY") - marker_params_a: Final = LiteLLMParamsBody( - model="auto_router/complexity_router", - complexity_router_config=_uniform_tier_config(deployments.tier_a), - tags=[deployments.tag_a], + marker_registration: Final = ( + named.shared, + LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config=_uniform_tier_config(named.tier), + tags=[named.tag], + ), ) - marker_params_b: Final = LiteLLMParamsBody( - model="auto_router/complexity_router", - complexity_router_config=_uniform_tier_config(deployments.tier_b), - tags=[deployments.tag_b], + tier_registration: Final = ( + named.tier, + LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key, tags=None if marker_first else [named.tag]), ) - registrations: Final[tuple[tuple[str, LiteLLMParamsBody], ...]] = ( - (deployments.shared_a, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)), - (deployments.tier_a, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key, tags=[deployments.tag_a])), - (deployments.shared_a, marker_params_a), - (deployments.shared_b, marker_params_b), - (deployments.tier_b, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=anthropic_key)), - (deployments.shared_b, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)), + plain_registration: Final = (named.shared, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=anthropic_key)) + registrations: Final = ( + (marker_registration, tier_registration, plain_registration) + if marker_first + else (plain_registration, tier_registration, marker_registration) ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield deployments - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, stack, registrations) + return named -@pytest.fixture(scope="module") -def zero_priced_alias(proxy: ProxyClient) -> Iterator[ZeroPricedAlias]: +@pytest.fixture(scope="class") +def plain_first_split(proxy: ProxyClient, router_stack: ExitStack) -> TagSplitDeployment: + return _tag_split(proxy, router_stack, marker_first=False) + + +@pytest.fixture(scope="class") +def marker_first_split(proxy: ProxyClient, router_stack: ExitStack) -> TagSplitDeployment: + return _tag_split(proxy, router_stack, marker_first=True) + + +@pytest.fixture(scope="class") +def zero_priced_alias(proxy: ProxyClient, router_stack: ExitStack) -> ZeroPricedAlias: marker: Final = unique_marker() named: Final = ZeroPricedAlias(alias=f"e2e-priced-alias-{marker}", tier=f"e2e-priced-tier-{marker}") alias_params: Final = LiteLLMParamsBody( @@ -263,16 +268,12 @@ def zero_priced_alias(proxy: ProxyClient) -> Iterator[ZeroPricedAlias]: (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), (named.alias, alias_params), ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield named - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, router_stack, registrations) + return named -@pytest.fixture(scope="module") -def heuristic_split(proxy: ProxyClient) -> Iterator[HeuristicSplit]: +@pytest.fixture(scope="class") +def heuristic_split(proxy: ProxyClient, router_stack: ExitStack) -> HeuristicSplit: marker: Final = unique_marker() named: Final = HeuristicSplit( alias=f"e2e-heuristic-router-{marker}", @@ -289,16 +290,12 @@ def heuristic_split(proxy: ProxyClient) -> Iterator[HeuristicSplit]: (named.strong, LiteLLMParamsBody(model=STRONG_MODEL, api_key=_provider_key("OPENAI_API_KEY"))), (named.alias, LiteLLMParamsBody(model="auto_router/complexity_router", complexity_router_config=config)), ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield named - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, router_stack, registrations) + return named -@pytest.fixture(scope="module") -def semantic_auto_router(proxy: ProxyClient) -> Iterator[SemanticAutoRouter]: +@pytest.fixture(scope="class") +def semantic_auto_router(proxy: ProxyClient, router_stack: ExitStack) -> SemanticAutoRouter: marker: Final = unique_marker() named: Final = SemanticAutoRouter( marker=f"e2e-semantic-router-{marker}", @@ -321,16 +318,12 @@ def semantic_auto_router(proxy: ProxyClient) -> Iterator[SemanticAutoRouter]: (named.fallback, LiteLLMParamsBody(model=PLAIN_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), (named.marker, marker_params), ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield named - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, router_stack, registrations) + return named -@pytest.fixture(scope="module") -def credentialed_alias(proxy: ProxyClient) -> Iterator[CredentialedAlias]: +@pytest.fixture(scope="class") +def credentialed_alias(proxy: ProxyClient, router_stack: ExitStack) -> CredentialedAlias: marker: Final = unique_marker() named: Final = CredentialedAlias(alias=f"e2e-cred-alias-{marker}", tier=f"e2e-cred-tier-{marker}") alias_params: Final = LiteLLMParamsBody( @@ -342,104 +335,110 @@ def credentialed_alias(proxy: ProxyClient) -> Iterator[CredentialedAlias]: (named.tier, LiteLLMParamsBody(model=CHEAP_MODEL, api_key=_provider_key("ANTHROPIC_API_KEY"))), (named.alias, alias_params), ) - created: Final = tuple(proxy.create_model(name, params) for name, params in registrations) - try: - yield named - finally: - for model_id in created: - proxy.delete_model(model_id) + _register_models(proxy, router_stack, registrations) + return named class TestTagSplitRouting: @pytest.mark.covers("reliability.routing.tagged_marker.request_tag_selects_marker") def test_body_tagged_chat_routes_through_the_marker_to_its_tier( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins GitHub issue #36619: with tag filtering on, a chat request whose body metadata tags match the tagged marker under a shared model name is answered by the marker's tier deployment, not by the plain deployment that was registered under the name first.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) - chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a, tags=[split.tag_a]))) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) + chat: Final = unwrap(proxy.chat(key, _hello_chat_body(plain_first_split.shared, tags=[plain_first_split.tag]))) assert chat.choices, "tagged chat through the shared name returned no choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged chat on the shared name") + _assert_served_only_by(rows, CHEAP_SERVED | {plain_first_split.tier}, "body-tagged chat on the shared name") @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") def test_untagged_chat_is_always_served_by_the_plain_deployment( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins GitHub issue #36620: untagged chat requests to the shared name succeed on every call and are all served by the plain deployment; the tagged marker never captures them, so no intermittent auto-router errors and no tier hijacking.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) for _ in range(5): - chat = unwrap(proxy.chat(key, _hello_chat_body(split.shared_a))) + chat = unwrap(proxy.chat(key, _hello_chat_body(plain_first_split.shared))) assert chat.choices, "untagged chat through the shared name returned no choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=5) - _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged chat on the shared name") + _assert_served_only_by(rows, PLAIN_SERVED | {plain_first_split.shared}, "untagged chat on the shared name") @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") def test_untagged_messages_is_served_by_the_plain_deployment( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins GitHub issue #36620 on the /v1/messages surface: an untagged Anthropic-native request to the shared name is served by the plain deployment, not captured by the tagged marker.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) - answer: Final = unwrap(proxy.messages(key, _hello_messages_body(split.shared_a))) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) + answer: Final = unwrap(proxy.messages(key, _hello_messages_body(plain_first_split.shared))) assert answer.content or answer.choices, "untagged /v1/messages returned neither content nor choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/messages on the shared name") + _assert_served_only_by( + rows, PLAIN_SERVED | {plain_first_split.shared}, "untagged /v1/messages on the shared name" + ) class TestUntaggedTierDeployments: @pytest.mark.covers("reliability.routing.tagged_marker.header_tag_selects_marker") def test_header_tagged_messages_routes_through_the_marker_to_an_untagged_tier( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, marker_first_split: TagSplitDeployment ) -> None: """Pins GitHub issue #36621: a /v1/messages request tagged only via the x-litellm-tags header selects the tagged marker, and the rewrite still lands on the tier deployment even though that deployment carries no tags, because the marker consumed the routing tags.""" - key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True) - headers: Final = TaggedAnthropicHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_b) + key: Final = _key_for( + proxy, resources, [marker_first_split.shared, marker_first_split.tier], tag_filtering=True + ) + headers: Final = TaggedAnthropicHeaders(authorization=f"Bearer {key}", x_litellm_tags=marker_first_split.tag) answer: Final = unwrap( proxy.transport.post( "/v1/messages", headers=headers, - json=_hello_messages_body(split.shared_b), + json=_hello_messages_body(marker_first_split.shared), response_type=AnthropicMessagesResponse, ) ) assert answer.content or answer.choices, "header-tagged /v1/messages returned neither content nor choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "header-tagged /v1/messages on the shared name") + _assert_served_only_by( + rows, CHEAP_SERVED | {marker_first_split.tier}, "header-tagged /v1/messages on the shared name" + ) @pytest.mark.covers("reliability.routing.tagged_marker.untagged_tier_deployments_still_served") def test_body_tagged_chat_reaches_the_untagged_tier_after_marker_rewrite( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, marker_first_split: TagSplitDeployment ) -> None: """Pins the tag-consumption half of GitHub issue #36621: after the tagged marker rewrites the request to its tier model, the consumed routing tags no longer constrain deployment selection, so the untagged tier deployment serves the request instead of a strict-tag denial.""" - key: Final = _key_for(proxy, resources, [split.shared_b, split.tier_b], tag_filtering=True) - chat: Final = unwrap(proxy.chat(key, _hello_chat_body(split.shared_b, tags=[split.tag_b]))) + key: Final = _key_for( + proxy, resources, [marker_first_split.shared, marker_first_split.tier], tag_filtering=True + ) + chat: Final = unwrap( + proxy.chat(key, _hello_chat_body(marker_first_split.shared, tags=[marker_first_split.tag])) + ) assert chat.choices, "body-tagged chat through the marker-first shared name returned no choices" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_b}, "body-tagged chat with untagged tier") + _assert_served_only_by(rows, CHEAP_SERVED | {marker_first_split.tier}, "body-tagged chat with untagged tier") @pytest.mark.covers("reliability.routing.tagged_marker.tag_semantics_stay_strict") def test_tagged_call_straight_at_an_untagged_deployment_stays_denied( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, marker_first_split: TagSplitDeployment ) -> None: """The tag-consumption fix must not loosen strict tag semantics: a tagged request aimed directly at an untagged deployment (no marker involved) is still rejected with the 401 tags-configuration error.""" - key: Final = _key_for(proxy, resources, [split.tier_b], tag_filtering=True) - result: Final = proxy.chat(key, _hello_chat_body(split.tier_b, tags=[split.tag_b])) + key: Final = _key_for(proxy, resources, [marker_first_split.tier], tag_filtering=True) + result: Final = proxy.chat(key, _hello_chat_body(marker_first_split.tier, tags=[marker_first_split.tag])) assert isinstance(result, UnauthorizedError), ( f"expected the tagged direct call to an untagged deployment to be denied with 401, got {result}" ) @@ -451,37 +450,39 @@ class TestUntaggedTierDeployments: class TestResponsesApiTagRouting: @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker") def test_header_tagged_responses_with_string_input_routes_to_the_tier( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins the /v1/responses surface of the tag split (GitHub issues #36620/#36621): a /v1/responses request with string input, tagged via the x-litellm-tags header, succeeds and routes through the tagged marker to its tier.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) - headers: Final = TaggedAuthHeaders(authorization=f"Bearer {key}", x_litellm_tags=split.tag_a) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) + headers: Final = TaggedAuthHeaders(authorization=f"Bearer {key}", x_litellm_tags=plain_first_split.tag) body: Final = ResponsesBody( - model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64 + model=plain_first_split.shared, input=f"say hello {unique_marker()}", max_output_tokens=64 ) answer: Final = unwrap( proxy.transport.post("/v1/responses", headers=headers, json=body, response_type=ResponsesApiResponse) ) assert answer.id, "header-tagged /v1/responses returned no response id" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "header-tagged /v1/responses string input") + _assert_served_only_by( + rows, CHEAP_SERVED | {plain_first_split.tier}, "header-tagged /v1/responses string input" + ) @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker") def test_body_tagged_responses_with_list_input_routes_to_the_tier( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins the body-tag and list-input combination of the same split: /v1/responses with litellm_metadata.tags and structured input items routes through the tagged marker to its tier.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) body: Final = ResponsesBody( - model=split.shared_a, + model=plain_first_split.shared, input=[ResponsesInputItem(role="user", content=f"say hello {unique_marker()}")], max_output_tokens=64, - litellm_metadata=ResponsesTagMetadata(tags=[split.tag_a]), + litellm_metadata=ResponsesTagMetadata(tags=[plain_first_split.tag]), ) answer: Final = unwrap( proxy.transport.post( @@ -493,18 +494,18 @@ class TestResponsesApiTagRouting: ) assert answer.id, "body-tagged /v1/responses returned no response id" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, CHEAP_SERVED | {split.tier_a}, "body-tagged /v1/responses list input") + _assert_served_only_by(rows, CHEAP_SERVED | {plain_first_split.tier}, "body-tagged /v1/responses list input") @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") def test_untagged_responses_is_served_by_the_plain_deployment( - self, proxy: ProxyClient, resources: ResourceManager, split: TagSplitDeployments + self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: """Pins the untagged half of the /v1/responses tag split: an untagged request to the shared name is served by the plain deployment, matching the chat and messages surfaces.""" - key: Final = _key_for(proxy, resources, [split.shared_a, split.tier_a], tag_filtering=True) + key: Final = _key_for(proxy, resources, [plain_first_split.shared, plain_first_split.tier], tag_filtering=True) body: Final = ResponsesBody( - model=split.shared_a, input=f"say hello {unique_marker()}", max_output_tokens=64 + model=plain_first_split.shared, input=f"say hello {unique_marker()}", max_output_tokens=64 ) answer: Final = unwrap( proxy.transport.post( @@ -516,7 +517,9 @@ class TestResponsesApiTagRouting: ) assert answer.id, "untagged /v1/responses returned no response id" rows: Final = proxy.poll_logs_for_key(key, min_rows=1) - _assert_served_only_by(rows, PLAIN_SERVED | {split.shared_a}, "untagged /v1/responses on the shared name") + _assert_served_only_by( + rows, PLAIN_SERVED | {plain_first_split.shared}, "untagged /v1/responses on the shared name" + ) class TestStrategyAliasPricing: @@ -551,9 +554,7 @@ class TestComplexityHeuristicScope: while the accompanying ~2KB agent system prompt is packed with enough reasoning and complexity keywords that scoring the combined text lands in REASONING; only ask-only scoring keeps this on the cheap tier.""" - key: Final = _key_for( - proxy, resources, [heuristic_split.alias, heuristic_split.cheap, heuristic_split.strong] - ) + key: Final = _key_for(proxy, resources, [heuristic_split.alias, heuristic_split.cheap, heuristic_split.strong]) body: Final = ChatBody( model=heuristic_split.alias, messages=[ diff --git a/tests/e2e/test_e2e_http.py b/tests/e2e/test_e2e_http.py index 66841725d1d..81cd6c8d3d1 100644 --- a/tests/e2e/test_e2e_http.py +++ b/tests/e2e/test_e2e_http.py @@ -13,13 +13,24 @@ monkeypatches anything. from __future__ import annotations from collections.abc import Callable, Iterator, Mapping, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass from types import MappingProxyType from typing import Final import pytest - -from e2e_http import RETRY_ATTEMPTS, TRANSIENT_STATUSES, request_with_retry, streaming_outcome +from e2e_http import ( + RETRY_ATTEMPTS, + TRANSIENT_STATUSES, + NoBody, + PartialBody, + Success, + ValidationError, + classify, + request_with_retry, + streaming_outcome, + wire_body, +) +from pydantic import BaseModel, TypeAdapter @dataclass @@ -33,10 +44,10 @@ class FakeResponse: @dataclass class SleepRecorder: - delays: list[float] = field(default_factory=list) + delays: tuple[float, ...] = () def __call__(self, seconds: float) -> None: - self.delays.append(seconds) + self.delays += (seconds,) def _issue_from(responses: Sequence[FakeResponse]) -> Callable[[], FakeResponse]: @@ -55,7 +66,7 @@ class TestTransientRetryPolicy: sleep = SleepRecorder() result = request_with_retry(_issue_from(responses), sleep=sleep) assert result is responses[0] - assert sleep.delays == [] + assert sleep.delays == () assert responses[0].close_calls == 0 def test_429_is_never_retried(self) -> None: @@ -63,7 +74,7 @@ class TestTransientRetryPolicy: sleep = SleepRecorder() result = request_with_retry(_issue_from(responses), sleep=sleep) assert result is responses[0] - assert sleep.delays == [] + assert sleep.delays == () assert responses[0].close_calls == 0 def test_overloaded_529_retries_with_backoff_then_returns_the_success(self) -> None: @@ -71,7 +82,7 @@ class TestTransientRetryPolicy: sleep = SleepRecorder() result = request_with_retry(_issue_from(responses), sleep=sleep) assert result is responses[1] - assert sleep.delays == [0.5] + assert sleep.delays == (0.5,) assert responses[0].close_calls == 1 assert responses[1].close_calls == 0 @@ -80,7 +91,7 @@ class TestTransientRetryPolicy: sleep = SleepRecorder() result = request_with_retry(_issue_from(responses), sleep=sleep) assert result is responses[RETRY_ATTEMPTS - 1] - assert sleep.delays == [0.5, 1.0] + assert sleep.delays == (0.5, 1.0) assert [r.close_calls for r in responses] == [1, 1, 0, 0] @@ -134,3 +145,65 @@ class TestStreamEventArrivals: assert result.stream_events == [] assert result.stream_event_arrivals == [] assert result.body == "bad request" + + +class _ServerUpdate(PartialBody): + server_id: str + alias: str | None = None + description: str | None = None + + +class _ServerCreate(BaseModel): + alias: str + description: str | None = None + + +class TestWireBody: + """A partial-update body must put exactly the caller's choice on the wire: an + omitted field stays off it so the route keeps the stored value, and an explicit + None goes out as JSON null so the route clears it. Plain bodies keep dropping + None, which is what every create route expects.""" + + def test_partial_body_omits_unset_fields_and_sends_explicit_none_as_null(self) -> None: + assert wire_body(_ServerUpdate(server_id="s1", description=None)) == {"server_id": "s1", "description": None} + assert wire_body(_ServerUpdate(server_id="s1", alias="renamed")) == {"server_id": "s1", "alias": "renamed"} + + def test_plain_body_drops_none_fields(self) -> None: + assert wire_body(_ServerCreate(alias="a", description=None)) == {"alias": "a"} + + +_JSON: Final[TypeAdapter[object]] = TypeAdapter(object) + + +@dataclass +class FakeJsonResponse: + """The `classify` view of a response: a status, the raw body bytes, and the + parse that would raise on an empty one.""" + + status_code: int + content: bytes + + @property + def ok(self) -> bool: + return self.status_code < 400 + + @property + def text(self) -> str: + return self.content.decode() + + def json(self) -> object: + return _JSON.validate_json(self.content) + + +class TestClassifyEmptyBody: + """A delete that answers 202 with no body is a success, not a parse failure: + the MCP server and toolset delete routes both answer that way, and reading it + as a failure would hide a delete that did not happen behind one that did.""" + + def test_empty_2xx_body_is_a_success(self) -> None: + result: Final = classify(FakeJsonResponse(status_code=202, content=b""), NoBody) + assert isinstance(result, Success) and result.status_code == 202 + + def test_body_that_is_not_json_is_still_a_validation_failure(self) -> None: + result: Final = classify(FakeJsonResponse(status_code=200, content=b""), NoBody) + assert isinstance(result, ValidationError) diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py index 2caac58333f..3b84a47e3cc 100644 --- a/tests/e2e/test_proxy_client.py +++ b/tests/e2e/test_proxy_client.py @@ -15,28 +15,35 @@ from collections.abc import Iterable, Mapping from dataclasses import dataclass from itertools import chain, repeat from types import MappingProxyType -from typing import Final +from typing import Final, cast import pytest - from e2e_config import parse_replica_urls from e2e_http import Result, Success from models import KeyInfo, KeyInfoResponse, ModelListEntry, ModelsListResponse from proxy_client import ( - Poller, ConvergeOutcome, Converged, + EverywhereConverged, ModelsPoller, + NeverConvergedOn, NotConverged, NotServableOn, + Poller, + ProxyClient, + ReplicaRead, Servable, await_converged_everywhere, + await_everywhere, await_servable_everywhere, - first_lagging_replica, + build_proxy_client, converge_timeout_message, + first_lagging_replica, ) +from transport import Transport MODEL: Final = "gpt-under-test" +_NO_TRANSPORTS: Final = cast(Transport, None) TIMEOUT: Final = 10.0 INTERVAL: Final = 2.0 RPM_BEFORE_UPDATE: Final = 100 @@ -187,3 +194,83 @@ class TestParseReplicaUrls: def test_falls_back_to_the_data_plane_address_when_unset(self) -> None: assert parse_replica_urls("", "http://lb") == ("http://lb",) + + +def _answers(answers: Iterable[str]) -> ReplicaRead[str]: + it: Final = iter(answers) + return lambda _timeout: next(it) + + +def _await_everywhere(reads: Mapping[str, ReplicaRead[str]]) -> EverywhereConverged[str] | NeverConvergedOn[str]: + clock: Final = FakeClock() + return await_everywhere( + reads, + settled=lambda answer: answer == "renamed", + timeout=TIMEOUT, + interval=INTERVAL, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + +class TestAwaitEverywhere: + def test_waits_for_the_lagging_replica_and_returns_every_settled_answer(self) -> None: + reads: Final = { + "gateway-1": _answers(repeat("renamed")), + "gateway-2": _answers(chain(repeat("stale", 2), repeat("renamed"))), + } + outcome: Final = _await_everywhere(reads) + assert isinstance(outcome, EverywhereConverged) + assert dict(outcome.answers) == {"gateway-1": "renamed", "gateway-2": "renamed"} + + def test_names_the_replica_that_never_converges_with_what_it_last_served(self) -> None: + reads: Final = { + "gateway-1": _answers(repeat("renamed")), + "gateway-2": _answers(repeat("stale")), + } + assert _await_everywhere(reads) == NeverConvergedOn(replica="gateway-2", last="stale") + + def test_polls_until_the_deadline_before_giving_up(self) -> None: + lagging: Final = chain(repeat("stale", int(TIMEOUT / INTERVAL)), repeat("renamed")) + outcome: Final = _await_everywhere({"gateway-1": _answers(lagging)}) + assert isinstance(outcome, EverywhereConverged), outcome + + +class TestReplicasFor: + def test_split_deployment_reads_management_routes_back_from_the_control_plane(self) -> None: + client: Final = build_proxy_client( + base_url="http://lb", + control_plane_base_url="http://backend", + replica_urls=("http://gateway-1", "http://gateway-2"), + ) + assert set(client.replicas_for("/key/info")) == {"http://backend"} + assert set(client.replicas_for("/v1/models")) == {"http://gateway-1", "http://gateway-2"} + + def test_monolith_reads_management_routes_back_from_every_replica(self) -> None: + client: Final = build_proxy_client( + base_url="http://lb", + control_plane_base_url="http://lb", + replica_urls=("http://pod-1", "http://pod-2"), + ) + assert set(client.replicas_for("/key/info")) == {"http://pod-1", "http://pod-2"} + + def test_mcp_admin_routes_read_back_from_every_data_plane_replica(self) -> None: + """/v1/mcp/* is a lazily mounted feature, so a data-plane replica serves it + too and answers from its own in-memory registry. Routing it to the control + plane would leave every replica but that one unproven, and would move the + tools/list barrier in mcp_client off the plane that serves tools/list.""" + client: Final = build_proxy_client( + base_url="http://lb", + control_plane_base_url="http://backend", + replica_urls=("http://gateway-1", "http://gateway-2"), + ) + assert set(client.replicas_for("/v1/mcp/server/abc")) == {"http://gateway-1", "http://gateway-2"} + assert set(client.replicas_for("/v1/mcp/toolset/abc")) == {"http://gateway-1", "http://gateway-2"} + + def test_a_route_no_replica_serves_is_refused_rather_than_read_back_vacuously(self) -> None: + """A read-back over zero replicas would satisfy every predicate and assert + nothing, so asking for one fails instead of passing silently.""" + client: Final = ProxyClient(transport=_NO_TRANSPORTS, replicas={}, control_replicas={}) + with pytest.raises(AssertionError, match="no replica is configured"): + _ = client.replicas_for("/v1/models") diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_partial_update.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_partial_update.py index c0d055edb7f..669e094fee4 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_partial_update.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_partial_update.py @@ -1,10 +1,11 @@ """ -Tests for partial-update semantics of PUT /v1/mcp/server. +Tests for partial-update semantics of PUT /v1/mcp/server and PUT /v1/mcp/toolset. A partial update must only write the fields the caller explicitly provided. Omitting a field must NOT reset it to its Pydantic schema default (e.g. ``transport=sse``, ``mcp_access_groups=[]``, ``allow_all_keys=False``), which -would silently overwrite the existing DB row. +would silently overwrite the existing DB row, and a field the caller sent as null +must be cleared rather than left at its stored value. """ import json @@ -850,3 +851,69 @@ async def test_cf_pair_switch_does_not_clear_dcr_bridge(): data = UpdateMCPServerRequest(server_id="s", auth_type="oauth_delegate") data_dict = await _run_update_with_existing(data, existing_auth_type="true_passthrough") assert "dcr_bridge" not in data_dict + + +def _mock_toolset_prisma(): + """A prisma double whose update answers with a row the reader can expand, so the + call under test returns instead of failing inside the row mapper.""" + updated_row = MagicMock() + updated_row.model_dump.return_value = { + "toolset_id": "ts-1", + "toolset_name": "ops", + "description": None, + "tools": "[]", + } + mock_prisma = MagicMock() + mock_prisma.db.litellm_mcptoolsettable = AsyncMock() + mock_prisma.db.litellm_mcptoolsettable.update = AsyncMock(return_value=updated_row) + return mock_prisma + + +async def _run_toolset_update(payload: dict) -> dict: + """The columns PUT /v1/mcp/toolset writes for this payload, minus the audit stamp + every write carries. The prisma double is injected, so nothing is patched.""" + from litellm.proxy._experimental.mcp_server.toolset_db import update_mcp_toolset + from litellm.types.mcp_server.mcp_toolset import UpdateMCPToolsetRequest + + mock_prisma = _mock_toolset_prisma() + await update_mcp_toolset(mock_prisma, UpdateMCPToolsetRequest.model_validate(payload), "test-user") + written = dict(mock_prisma.db.litellm_mcptoolsettable.update.call_args[1]["data"]) + assert written["updated_by"] == "test-user" + return {name: value for name, value in written.items() if name != "updated_by"} + + +@pytest.mark.asyncio +async def test_toolset_partial_update_clears_description_on_explicit_null(): + """The dump used to drop None, so a null description could never clear the stored + one: the toolset kept a description its owner had deleted.""" + assert await _run_toolset_update({"toolset_id": "ts-1", "description": None}) == {"description": None} + + +@pytest.mark.asyncio +async def test_toolset_partial_update_omits_the_fields_the_caller_left_out(): + tools = [{"server_id": "s1", "tool_name": "alpha"}] + assert await _run_toolset_update({"toolset_id": "ts-1", "tools": tools}) == {"tools": json.dumps(tools)} + + +@pytest.mark.asyncio +async def test_toolset_partial_update_ignores_null_tools_rather_than_revoking_them(): + """A client that sends tools=null means "leave the selection alone", so the grants + survive. Clearing them is an explicit [], which cannot be confused with an omitted + field; treating null as a clear would silently revoke every tool the toolset grants.""" + assert await _run_toolset_update({"toolset_id": "ts-1", "tools": None, "description": "kept"}) == { + "description": "kept" + } + + +@pytest.mark.asyncio +async def test_toolset_partial_update_empties_the_selection_on_an_explicit_empty_list(): + assert await _run_toolset_update({"toolset_id": "ts-1", "tools": []}) == {"tools": "[]"} + + +@pytest.mark.asyncio +async def test_toolset_partial_update_ignores_a_null_name(): + """A toolset always has a name, so a null toolset_name is a no-op, not a clear + that would write a NOT NULL column to null.""" + assert await _run_toolset_update({"toolset_id": "ts-1", "toolset_name": None, "description": "kept"}) == { + "description": "kept" + } From e8e3172d7d70558929f32f057ebe4c7471c8c352 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 8 Sep 2026 23:10:18 -0700 Subject: [PATCH 177/411] fix(model-management): honor an explicit null as a clear on model update (#40047) * fix(model-management): honor an explicit null as a clear on model update PATCH /model/{model_id}/update merged the patch with exclude_none and then popped explicit nulls only for the mirrored pricing fields, so a null sent for max_input_tokens, mode, supports_vision or any other key was dropped and a value pinned by an earlier save could never be removed. The route now follows JSON Merge Patch over both blobs: a key absent from the body is unchanged, a key sent as null is removed from the stored row, and a key sent with a value is set. Ownership and identity keys keep ignoring a null, as do the fields the stored models require, since clearing one writes a row no reload can rebuild. Mirrored pricing keys still clear from both blobs. Clearing a price also needed the router to stop merging a deployment's cost-map entry onto its previous registration, which left the old rate in place and kept billing at a price the deployment no longer carried. Adds a create, read, partial-update, clear, enforce, delete lifecycle e2e that reads back on every replica, and a harness helper for that read-back. * fix(router): keep a deployment id that names a real model from evicting its catalog entry Deployments are keyed into litellm.model_cost alongside the built-in catalog, so evicting a deployment's stale entry by id could take a real model's entry with it: registering a deployment whose model_info.id is "gpt-4o" stripped that model's pricing, context window and capability flags process-wide, for every other deployment of it, until the next price-map reload. Only evict an entry this registration owns. A colliding id keeps the previous merge, which pollutes the catalog entry rather than emptying it. Also pins the Admin UI round trip: the model edit form echoes the whole /model/info row back on save, and that read reports every key the deployment never stored as an explicit null, so the clear path has to leave those keys alone. * fix(router): decide cost-map eviction by what this registrar created The previous guard read a catalog entry off `litellm_provider`, so a deployment that declares its own provider in model_info was treated as one and kept billing at a price it no longer carried. It also only held for a single registration: a second one under a colliding id saw the id the first merge left behind and evicted the catalog entry anyway. Track the cost-map keys this registrar creates instead. A key it created is evicted before re-registration; one it did not is left to merge, which is what a deployment id colliding with a catalog model name needs. Also folds the required-fields comment into the docstring that already gives the reason. * fix(router): release a deployment's cost-map key when it is deleted The ownership ledger only grew. A deleted deployment kept its claim, so if a later catalog refresh started publishing a model under that same name, the next registration would treat the catalog entry as the deployment's own and evict it. Deleting a deployment now gives the key back, which also stops the ledger growing for the life of the process. * fix(router): hold a cost-map key while another live router still serves it The claim is process-wide but the release was per-deletion, so with two routers serving one deployment id, the first deletion put the survivor back on merging and the price it had just cleared would keep billing. Release the key only once no live router still serves that id. * fix(router): register a router in the live set when it gains a deployment _live_routers was only joined when a router was constructed with a model_list, but a router built empty is populated through add_deployment, and the empty branch exists for exactly that. Such a router was invisible to the live-router scan, so deleting the deployment from another router released the shared cost-map key while it was still serving that id. Joining the set where a deployment enters the list covers every path, and it also lets a price reload rebuild what a dynamically built router serves. * fix(e2e): read the stored model row from the control plane, not each gateway The lifecycle suite polled /model/info on every URL in PROXY_REPLICA_URLS. Those URLs are the stack's gateways, and gateway/routes/allowlist.py trims them to the LLM data-plane surface, so /model/info answers only on the backend and 404s on every replica. All five tests failed at their first read-back in CI while passing against a monolith, where one process serves both planes. The stored row has one answer behind it, so it is read through the shared transport, which routes control-plane paths to the backend. What every gateway must agree on is which models it serves, so the create and delete steps poll /v1/models per replica instead, a route the gateway does serve. read_back_everywhere now rejects a control-plane path outright rather than timing out on it. Two things surfaced behind that. /public/ was missing from the transport's control-plane prefixes, so model_cost_map() was routed to a gateway and 404'd, and the billing steps needed a data-plane wait: a PATCH lands on the backend and each gateway picks it up on its own config reload, measured here at 12-24s, so they now drive calls until the new rate reaches the spend row and let the deadline fail them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1S92J8gSxxKVe1JBzxWBF * test(models): keep polling outcomes immutable and document shared ownership * test: validate opaque stream IDs and hide log-reader credentials * test: isolate auto-router scenarios and clean partial setup --------- Co-authored-by: Claude Opus 5 --- .../model_management_endpoints.py | 70 +++- litellm/router.py | 26 +- tests/e2e/coverage_registry/mgmt.yaml | 2 + .../management/test_model_lifecycle_e2e.py | 365 ++++++++++++++++++ tests/e2e/models.py | 74 +++- tests/e2e/proxy_client.py | 199 +++++++++- tests/e2e/test_proxy_client.py | 57 ++- tests/e2e/transport.py | 1 + .../test_model_management_endpoints.py | 178 ++++++++- .../test_router_model_cost_isolation.py | 192 +++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 +- 11 files changed, 1130 insertions(+), 39 deletions(-) create mode 100644 tests/e2e/management/test_model_lifecycle_e2e.py diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 0f19e9ce149..742b9d9817f 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -119,6 +119,7 @@ from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, Deployment, GenericLiteLLMParams, + LiteLLM_Params, ModelInfo, updateDeployment, ) @@ -728,6 +729,44 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) +_OWNERSHIP_FIELDS: Final = frozenset( + { + "db_model", + "team_id", + "team_public_model_name", + "access_groups", + "created_at", + "created_by", + "updated_at", + "updated_by", + "blocked", + } +) + +_STORED_REQUIRED_FIELDS: Final = frozenset( + name for model in (LiteLLM_Params, ModelInfo) for name, field in model.model_fields.items() if field.is_required() +) + +_NULL_CLEAR_IGNORED_FIELDS: Final = _OWNERSHIP_FIELDS | _STORED_REQUIRED_FIELDS | frozenset(PTU_MODEL_INFO_FIELDS) + + +def _explicitly_cleared_fields(patch: BaseModel | None) -> frozenset[str]: + """The keys a patch sends as an explicit null, which update_db_model removes from the + stored blob (JSON Merge Patch). Ownership keys are left alone, as are the keys the stored + models require, since clearing one writes a row no reload can rebuild through + LiteLLM_Params / ModelInfo. The PTU keys are handled by _explicitly_cleared_ptu_fields, + whose clear is gated on the feature flag. Applied after both blobs merge, so a model_info blob + the UI echoes back cannot resurrect a pricing key the litellm_params patch clears. + """ + if patch is None: + return frozenset() + return frozenset( + field + for field in patch.model_fields_set + if field not in _NULL_CLEAR_IGNORED_FIELDS and getattr(patch, field) is None + ) + + def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) @@ -748,25 +787,15 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr if updated_patch.model_info: merged_model_info.update(updated_patch.model_info.model_dump(exclude_none=True)) - # Honor explicit-null clears LAST, after both merges, so a model_info blob the UI - # passes through (which today re-sends the OLD pricing on every save) cannot - # silently undo a litellm_params clear via .update(). - # - # Restricted to SPECIAL_MODEL_INFO_PARAMS (input/output cost per token/character - # and cache read/write costs) so this path cannot be used to null out privileged - # model_info fields like team_id or access groups. SPECIAL_MODEL_INFO_PARAMS are - # mirrored between litellm_params and model_info by Deployment.__init__, so the - # clear propagates to both blobs. - if updated_patch.litellm_params: - for field in updated_patch.litellm_params.model_fields_set: - if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.litellm_params, field) is None: - merged_litellm_params.pop(field, None) - merged_model_info.pop(field, None) + for field in _explicitly_cleared_fields(updated_patch.litellm_params): + merged_litellm_params.pop(field, None) + if field in SPECIAL_MODEL_INFO_PARAMS: + merged_model_info.pop(field, None) + for field in _explicitly_cleared_fields(updated_patch.model_info): + merged_model_info.pop(field, None) + if field in SPECIAL_MODEL_INFO_PARAMS: + merged_litellm_params.pop(field, None) if updated_patch.model_info: - for field in updated_patch.model_info.model_fields_set: - if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.model_info, field) is None: - merged_model_info.pop(field, None) - merged_litellm_params.pop(field, None) for field in _explicitly_cleared_ptu_fields(updated_patch.model_info): merged_model_info.pop(field, None) @@ -816,8 +845,9 @@ async def patch_model( """ PATCH Endpoint for partial model updates. - Only updates the fields specified in the request while preserving other existing values. - Follows proper PATCH semantics by only modifying provided fields. + JSON Merge Patch semantics over `litellm_params` and `model_info`: a key absent from the + body is unchanged, a key sent as null is removed from the stored row, and a key sent with a + value is set (identity and ownership keys such as `id` and `team_id` ignore a null). Args: model_id: The ID of the model to update diff --git a/litellm/router.py b/litellm/router.py index 934a4ac86a9..1252d7e7487 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -628,6 +628,15 @@ RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( RETRY_BREADCRUMB_LIMIT: Final = 4 +# Cost-map keys created by _register_deployment_in_model_cost, which shares one flat +# namespace with the built-in model catalog. Only a key it created may be evicted, or a +# deployment whose id names a real model would strip that model's pricing and +# capabilities for every other deployment of it. delete_deployment gives a key back once no +# live router still serves that id, so a later catalog refresh that starts serving the name +# is not treated as a deployment's own. +_DEPLOYMENT_COST_MAP_KEYS: Final[set[str]] = set() # mutable-ok: ownership of shared cost-map keys + + class FallbackAwareStreamWrapper(CustomStreamWrapper): """Base for the Router's chat-completion stream wrappers, which are built around the attempt the Router picked first and have to repoint themselves when a fallback takes over.""" @@ -9761,6 +9770,7 @@ class Router: """ idx: Final = len(self.model_list) self.model_list.append(model) + _live_routers.add(self) # mutable-ok: track dynamic routers without extending their lifetimes self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() @@ -9929,7 +9939,12 @@ class Router: """Write a deployment's metadata into ``litellm.model_cost``. Runs when a deployment is added and again after a price data reload, so - the entries a refresh rebuilds are the ones a fresh boot would produce. + the entries a refresh rebuilds are the ones a fresh boot would produce. An + entry this function created is replaced rather than merged, so a price cleared + from the deployment does not linger from an earlier registration and keep + billing at the old rate. An entry it did not create is left to merge, because + a deployment id that collides with a catalog model name shares that model's + entry with every other deployment of it. Nothing is recorded for replay: a refresh walks the live routers instead, so a deleted, repointed or never-added deployment, and a discarded router, drop out of the rebuild on their own. @@ -9946,6 +9961,10 @@ class Router: } if model_id is not None: + if model_id in _DEPLOYMENT_COST_MAP_KEYS: + litellm.model_cost.pop(model_id, None) # mutable-ok: remove cleared prices from the shared entry + elif model_id not in litellm.model_cost: + _DEPLOYMENT_COST_MAP_KEYS.add(model_id) # mutable-ok: retain shared ownership across reloads litellm.register_model( model_cost={model_id: model_info}, persist_across_reloads=False, @@ -10042,6 +10061,11 @@ class Router: _budget_limiter: Final = self._get_router_deployment_budget_limiter() if _budget_limiter is not None: _budget_limiter.unregister_deployment_budget(model_id=id) + if not any( + router is not self and id in router.model_id_to_deployment_index_map + for router in tuple(_live_routers) + ): + _DEPLOYMENT_COST_MAP_KEYS.discard(id) # mutable-ok: the last owning router released this key try: self._unregister_pre_routing_strategy_for_deployment( deployment=item if isinstance(item, Deployment) else Deployment(**item) diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index c8d7037d2fd..83f1711a245 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -76,6 +76,8 @@ - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} - {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven} +- {id: mgmt.model.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "model_management_endpoints.py:731", rationale: "A partial PATCH changes only the keys it names; every other stored key reads back byte-for-byte, and the new rate reaches billing"} +- {id: mgmt.model.update.clear_persists, module: mgmt, tier: P0, surface: api, assertions: [clear_persists], source: "model_management_endpoints.py:731", fail_before_fix: proven, rationale: "An explicit null on PATCH removes the key from the stored row and billing falls back to the cost map; before the fix only mirrored pricing keys could be cleared"} - {id: mgmt.mcp_server.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1577", rationale: "Every field of an admin-created MCP server reads back verbatim, by id and in the list, on every replica"} - {id: mgmt.mcp_server.list.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1112", rationale: "The MCP page grid lists a created server with the same field values its detail view reports"} - {id: mgmt.mcp_server.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "mcp_management_endpoints.py:2665", rationale: "A dashboard edit of one field leaves the others intact and is visible on every replica after one save; edits that took several saves to stick were a customer defect"} diff --git a/tests/e2e/management/test_model_lifecycle_e2e.py b/tests/e2e/management/test_model_lifecycle_e2e.py new file mode 100644 index 00000000000..99044efeb46 --- /dev/null +++ b/tests/e2e/management/test_model_lifecycle_e2e.py @@ -0,0 +1,365 @@ +"""Live e2e: the lifecycle of a DB-stored deployment through the model management +routes, read back on every gateway replica. + +Each test registers its own gpt-4o-mini mock deployment through /model/new (deleted +on teardown) with non-default pricing, context window, mode, and api_base pinned, then +walks the lifecycle up to the step it proves: the create reads back field for field, +a partial PATCH changes only the key it names, an explicit null on PATCH removes the +key from the stored row (JSON Merge Patch), a call after the price clear is billed at +the cost map's rate rather than the cleared override, and a delete removes the +deployment from /model/info and makes the model name unknown to /chat/completions. + +The stored row is read back from /model/info, a control-plane route with one answer +behind it. What every gateway must agree on is which models it serves, so the create +and delete steps poll /v1/models on every URL in PROXY_REPLICA_URLS through +ProxyClient.read_model_back_everywhere, failing by name on the gateway that never converged. +""" + +from __future__ import annotations + +import math +import time +from collections.abc import Callable +from dataclasses import dataclass +from typing import Final + +import pytest +from pydantic import BaseModel + +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import ( + ChatBody, + ChatMessage, + Clear, + LiteLLMParamsBody, + LiteLLMParamsPatch, + ModelInfoBody, + ModelInfoEntry, + ModelInfoResponse, + ModelNewBody, + ModelPatchBody, + ModelsListResponse, + SpendLogRow, +) + +pytestmark = pytest.mark.e2e + +BACKEND_MODEL: Final = "gpt-4o-mini" +PINNED_API_BASE: Final = "https://pinned.example.invalid/v1" +PINNED_MAX_INPUT_TOKENS: Final = 4096 +PINNED_INPUT_RATE: Final = 1e-05 +UPDATED_INPUT_RATE: Final = 2e-05 +PINNED_OUTPUT_RATE: Final = 3e-05 + +# A PATCH lands on the control plane, and each gateway picks it up on its own config +# reload, so the first call after the write can still be billed at the old rate. There +# is no price on the gateway's data-plane surface to poll, so the billing steps drive +# calls until the new rate shows up in the spend row and let the deadline be what fails. +BILLING_CONVERGENCE_TIMEOUT: Final = 90.0 +BILLING_CONVERGENCE_INTERVAL: Final = 5.0 + + +@dataclass(frozen=True, slots=True) +class Registered: + model_name: str + model_id: str + + +class _ErrorDetail(BaseModel): + message: str + + +class _ErrorEnvelope(BaseModel): + error: _ErrorDetail + + +def _register(client: ManagementClient, resources: ResourceManager) -> Registered: + """Register a mock gpt-4o-mini deployment with every field under test pinned to a + non-default value, deleted on teardown. max_input_tokens is pinned in + litellm_params only: a value in model_info is copied into the shared cost-map + entry for the backend model, which would leak into every other gpt-4o-mini + deployment on the proxy.""" + model_name: Final = f"e2e-lifecycle-{unique_marker()}" + model_id: Final = client.proxy.register_model( + ModelNewBody( + model_name=model_name, + litellm_params=LiteLLMParamsBody( + model=BACKEND_MODEL, + mock_response="ok", + api_base=PINNED_API_BASE, + input_cost_per_token=PINNED_INPUT_RATE, + output_cost_per_token=PINNED_OUTPUT_RATE, + max_input_tokens=PINNED_MAX_INPUT_TOKENS, + ), + model_info=ModelInfoBody(mode="chat"), + ) + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return Registered(model_name=model_name, model_id=model_id) + + +def _entry(body: ModelInfoResponse, model_name: str) -> ModelInfoEntry | None: + return next((entry for entry in body.data if entry.model_name == model_name), None) + + +def _stored_entry( + client: ManagementClient, + model_name: str, + *, + converged: Callable[[ModelInfoEntry], bool], +) -> ModelInfoEntry: + """The stored /model/info row for `model_name`, once it satisfies `converged`. + + /model/info is a control-plane route: the gateways named in PROXY_REPLICA_URLS + serve the LLM surface only, so the stored row has one answer, not one per + gateway. What every gateway must agree on is which models it serves, and + `_assert_served_everywhere` / `_assert_absent_everywhere` poll /v1/models for + that.""" + + def has_converged(body: ModelInfoResponse) -> bool: + entry: Final = _entry(body, model_name) + return entry is not None and converged(entry) + + body: Final = client.proxy.read_model_back("/model/info", ModelInfoResponse, predicate=has_converged) + entry: Final = _entry(body, model_name) + assert entry is not None, f"/model/info stopped listing {model_name!r} between the poll and the read" + return entry + + +def _serves(body: ModelsListResponse, model_name: str) -> bool: + return any(entry.id == model_name for entry in body.data) + + +def _assert_served_everywhere(client: ManagementClient, model_name: str) -> None: + _ = client.proxy.read_model_back_everywhere( + "/v1/models", ModelsListResponse, predicate=lambda body: _serves(body, model_name) + ) + + +def _assert_absent_everywhere(client: ManagementClient, model_name: str) -> None: + _ = client.proxy.read_model_back_everywhere( + "/v1/models", ModelsListResponse, predicate=lambda body: not _serves(body, model_name) + ) + + +def _assert_untouched_keys_as_created(entry: ModelInfoEntry, replica: str) -> None: + """The keys no later step names read back byte-for-byte as /model/new wrote them.""" + params: Final = entry.litellm_params + assert params.model == BACKEND_MODEL, f"{replica}: litellm_params.model {params.model!r} != {BACKEND_MODEL!r}" + assert params.api_base == PINNED_API_BASE, f"{replica}: api_base {params.api_base!r} != {PINNED_API_BASE!r}" + assert params.output_cost_per_token == PINNED_OUTPUT_RATE, ( + f"{replica}: output_cost_per_token {params.output_cost_per_token} != {PINNED_OUTPUT_RATE}" + ) + assert entry.model_info.mode == "chat", f"{replica}: model_info.mode {entry.model_info.mode!r} != 'chat'" + + +def _approx_equal(actual: float, expected: float) -> bool: + return math.isclose(actual, expected, rel_tol=1e-2, abs_tol=1e-9) + + +def _priced(rows: list[SpendLogRow]) -> bool: + return any(row.metadata and row.metadata.cost_breakdown and row.metadata.cost_breakdown.input_cost for row in rows) + + +def _billed_input_cost(client: ManagementClient, model_name: str, key: str) -> tuple[int, float]: + """Drive one chat completion through `model_name` and return the prompt tokens and + input cost its spend row recorded, so a test can assert the rate the gateway actually + billed rather than only the rate it stored.""" + chat: Final = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model_name, + messages=[ChatMessage(role="user", content=f"reply with one word {unique_marker()}")], + max_tokens=16, + ), + ) + ) + assert chat.id is not None, f"chat completion carried no id to find its spend row by: {chat}" + + rows: Final = client.proxy.poll_logs_for_request_id(chat.id, predicate=_priced) + row: Final = next((row for row in rows if row.request_id == chat.id), None) + assert row is not None and row.metadata and row.metadata.cost_breakdown, ( + f"no priced spend row for request {chat.id} before the deadline: {rows}" + ) + prompt_tokens: Final = row.prompt_tokens or 0 + input_cost: Final = row.metadata.cost_breakdown.input_cost + assert prompt_tokens > 0 and input_cost is not None, f"spend row logged no prompt tokens or input cost: {row}" + return prompt_tokens, input_cost + + +def _await_billed_input_cost( + client: ManagementClient, model_name: str, key: str, *, expected_rate: float +) -> tuple[int, float]: + """Drive calls through `model_name` until one is billed at `expected_rate`, and + return the prompt tokens and input cost of the last spend row either way. + + Only the deadline ends the wait unsatisfied: a rate that never reaches the gateway + comes back as the stale cost for the caller to assert on, so the rate the caller + expects is still what decides the test.""" + deadline: Final = time.monotonic() + BILLING_CONVERGENCE_TIMEOUT + while True: + prompt_tokens, input_cost = _billed_input_cost(client, model_name, key) + if _approx_equal(input_cost, prompt_tokens * expected_rate) or time.monotonic() >= deadline: + return prompt_tokens, input_cost + time.sleep(BILLING_CONVERGENCE_INTERVAL) + + +class TestModelLifecycle: + @pytest.mark.covers("mgmt.model.add.persists") + def test_create_reads_back_every_field_and_serves_on_every_replica( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + registered = _register(client, resources) + + entry = _stored_entry(client, registered.model_name, converged=lambda _entry: True) + stored = "/model/info" + + _assert_untouched_keys_as_created(entry, stored) + assert entry.litellm_params.input_cost_per_token == PINNED_INPUT_RATE, ( + f"{stored}: input_cost_per_token {entry.litellm_params.input_cost_per_token} != {PINNED_INPUT_RATE}" + ) + assert entry.litellm_params.max_input_tokens == PINNED_MAX_INPUT_TOKENS, ( + f"{stored}: max_input_tokens {entry.litellm_params.max_input_tokens} != {PINNED_MAX_INPUT_TOKENS}" + ) + assert entry.model_info.id == registered.model_id, ( + f"{stored}: model_info.id {entry.model_info.id!r} != {registered.model_id!r}" + ) + + _assert_served_everywhere(client, registered.model_name) + + @pytest.mark.covers("mgmt.model.update.preserves_unrelated_fields") + def test_partial_update_changes_only_the_named_key( + self, client: ManagementClient, resources: ResourceManager, scoped_key: str + ) -> None: + registered = _register(client, resources) + + stored = client.proxy.patch_model( + registered.model_id, + ModelPatchBody(litellm_params=LiteLLMParamsPatch(input_cost_per_token=UPDATED_INPUT_RATE)), + ) + assert stored.litellm_params.input_cost_per_token == UPDATED_INPUT_RATE, ( + f"PATCH response stores input_cost_per_token {stored.litellm_params.input_cost_per_token}, " + f"sent {UPDATED_INPUT_RATE}" + ) + + entry = _stored_entry( + client, + registered.model_name, + converged=lambda entry: entry.litellm_params.input_cost_per_token == UPDATED_INPUT_RATE, + ) + stored = "/model/info" + + _assert_untouched_keys_as_created(entry, stored) + assert entry.litellm_params.max_input_tokens == PINNED_MAX_INPUT_TOKENS, ( + f"{stored}: max_input_tokens {entry.litellm_params.max_input_tokens} != {PINNED_MAX_INPUT_TOKENS}" + ) + assert entry.model_info.input_cost_per_token == UPDATED_INPUT_RATE, ( + f"{stored}: model_info.input_cost_per_token {entry.model_info.input_cost_per_token} " + f"did not mirror the updated {UPDATED_INPUT_RATE}" + ) + + prompt_tokens, input_cost = _await_billed_input_cost( + client, registered.model_name, scoped_key, expected_rate=UPDATED_INPUT_RATE + ) + assert _approx_equal(input_cost, prompt_tokens * UPDATED_INPUT_RATE), ( + f"input_cost {input_cost} != {prompt_tokens} tokens * updated rate {UPDATED_INPUT_RATE} " + f"= {prompt_tokens * UPDATED_INPUT_RATE}; the partial update did not reach billing" + ) + + @pytest.mark.covers("mgmt.model.update.clear_persists") + def test_explicit_null_removes_the_key_from_the_stored_row( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + registered = _register(client, resources) + + stored = client.proxy.patch_model( + registered.model_id, + ModelPatchBody(litellm_params=LiteLLMParamsPatch(max_input_tokens=Clear(), input_cost_per_token=Clear())), + ) + stored_params = stored.litellm_params.model_fields_set + assert "max_input_tokens" not in stored_params, ( + f"stored litellm_params still carries max_input_tokens " + f"{stored.litellm_params.max_input_tokens} after an explicit null" + ) + assert "input_cost_per_token" not in stored_params, ( + f"stored litellm_params still carries input_cost_per_token " + f"{stored.litellm_params.input_cost_per_token} after an explicit null" + ) + assert "max_input_tokens" not in stored.model_info.model_fields_set, ( + f"stored model_info carries max_input_tokens {stored.model_info.max_input_tokens} after the clear" + ) + assert "input_cost_per_token" not in stored.model_info.model_fields_set, ( + f"stored model_info still mirrors input_cost_per_token {stored.model_info.input_cost_per_token}" + ) + + cost_map_input_rate = client.proxy.model_cost_map()[BACKEND_MODEL].input_cost_per_token + assert cost_map_input_rate is not None, f"cost map has no input rate for {BACKEND_MODEL}" + entry = _stored_entry( + client, + registered.model_name, + converged=lambda entry: "max_input_tokens" not in entry.litellm_params.model_fields_set, + ) + stored = "/model/info" + + _assert_untouched_keys_as_created(entry, stored) + served = entry.litellm_params.model_fields_set + assert "max_input_tokens" not in served, ( + f"{stored}: litellm_params still serves max_input_tokens {entry.litellm_params.max_input_tokens}" + ) + assert "input_cost_per_token" not in served, ( + f"{stored}: litellm_params still serves input_cost_per_token {entry.litellm_params.input_cost_per_token}" + ) + assert entry.model_info.input_cost_per_token == cost_map_input_rate, ( + f"{stored}: model_info.input_cost_per_token {entry.model_info.input_cost_per_token} is not the " + f"cost map's {cost_map_input_rate}; the cleared override {PINNED_INPUT_RATE} still resolves" + ) + + @pytest.mark.covers("mgmt.model.update.clear_persists") + def test_cleared_price_is_billed_at_the_cost_map_rate( + self, client: ManagementClient, resources: ResourceManager, scoped_key: str + ) -> None: + registered = _register(client, resources) + _ = client.proxy.patch_model( + registered.model_id, + ModelPatchBody(litellm_params=LiteLLMParamsPatch(max_input_tokens=Clear(), input_cost_per_token=Clear())), + ) + _ = _stored_entry( + client, + registered.model_name, + converged=lambda entry: "input_cost_per_token" not in entry.litellm_params.model_fields_set, + ) + cost_map_input_rate = client.proxy.model_cost_map()[BACKEND_MODEL].input_cost_per_token + assert cost_map_input_rate is not None, f"cost map has no input rate for {BACKEND_MODEL}" + + prompt_tokens, input_cost = _await_billed_input_cost( + client, registered.model_name, scoped_key, expected_rate=cost_map_input_rate + ) + + assert _approx_equal(input_cost, prompt_tokens * cost_map_input_rate), ( + f"input_cost {input_cost} != {prompt_tokens} tokens * cost map rate {cost_map_input_rate} " + f"= {prompt_tokens * cost_map_input_rate}" + ) + assert not _approx_equal(input_cost, prompt_tokens * PINNED_INPUT_RATE), ( + f"input_cost {input_cost} is still billed at the cleared override {PINNED_INPUT_RATE}" + ) + + @pytest.mark.covers("mgmt.model.delete.persists") + def test_delete_removes_the_deployment_everywhere( + self, client: ManagementClient, resources: ResourceManager, scoped_key: str + ) -> None: + registered = _register(client, resources) + _ = _stored_entry(client, registered.model_name, converged=lambda _entry: True) + + client.delete_model_strict(registered.model_id) + + _assert_absent_everywhere(client, registered.model_name) + refused = client.chat_status(scoped_key, registered.model_name, "hi this is a test") + assert refused.status_code == 400, ( + f"chat against the deleted model must be rejected 400, got {refused.status_code}: {refused.body[:300]}" + ) + envelope = _ErrorEnvelope.model_validate_json(refused.body) + assert envelope.error.message, f"400 body must carry an error message: {refused.body[:300]}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 62810e6cfd9..8db37bd25a5 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -655,6 +655,11 @@ class OcrResponse(BaseModel): # ---------- spend logs ---------- +class CostBreakdown(BaseModel): + input_cost: float | None = None + output_cost: float | None = None + + class GuardrailEntityMatch(BaseModel): entity_type: str score: float @@ -672,6 +677,7 @@ class GuardrailRunRecord(BaseModel): class SpendLogMetadata(BaseModel): + cost_breakdown: CostBreakdown | None = None applied_guardrails: list[str] | None = None guardrail_information: list[GuardrailRunRecord] | None = None @@ -815,15 +821,42 @@ class CustomPricing(BaseModel): return prompt_tokens * self.input_cost_per_token + completion_tokens * self.output_cost_per_token +class DeploymentParams(CustomPricing): + """The litellm_params half of a /model/info row: the stored deployment as written, + credentials scrubbed. Unlike model_info it is never back-filled from the cost map, + so a key the store dropped is absent here (check `model_fields_set`).""" + + model: str | None = None + api_base: str | None = None + max_input_tokens: int | None = None + + +class DeploymentModelInfo(CustomPricing): + id: str | None = None + max_input_tokens: int | None = None + + class ModelInfoEntry(BaseModel): """One /model/info row. `litellm_params` is the configured deployment (carries any custom-pricing override); `model_info` is the price the proxy resolved for - it - the override merged over the cost-map defaults.""" + it - the override merged over the cost-map defaults, so a key cleared from the + stored blob reads as the cost-map default here.""" model_config = ConfigDict(protected_namespaces=()) model_name: str - litellm_params: CustomPricing = CustomPricing() - model_info: CustomPricing = CustomPricing() + litellm_params: DeploymentParams = DeploymentParams() + model_info: DeploymentModelInfo = DeploymentModelInfo() + + +class StoredDeployment(BaseModel): + """PATCH /model/{model_id}/update answers with the row as stored: both blobs raw, + nothing back-filled, so a cleared key is absent from `model_fields_set` of the + blob it was cleared from.""" + + model_config = ConfigDict(protected_namespaces=()) + model_name: str + litellm_params: DeploymentParams + model_info: DeploymentModelInfo class ModelInfoResponse(BaseModel): @@ -924,9 +957,10 @@ class LiteLLMParamsBody(BaseModel): timeout: float | None = None tpm: int | None = None weight: int | None = None + max_input_tokens: int | None = None -ModelMode = Literal["batch", "realtime", "image_generation"] +ModelMode = Literal["chat", "batch", "realtime", "image_generation"] class ModelInfoBody(BaseModel): @@ -936,6 +970,7 @@ class ModelInfoBody(BaseModel): # constraint when a prior run's teardown had not removed the row. id: str | None = None mode: ModelMode | None = None + max_input_tokens: int | None = None access_groups: list[str] | None = None team_id: str | None = None allowed_fails_policy: dict[str, int] | None = None @@ -964,6 +999,37 @@ class ModelUpdateBody(BaseModel): model_info: ModelInfoBody +class Clear(BaseModel): + """Serializes to JSON null. The transport dumps every body with exclude_none, so a + field set to this is how a patch carries the explicit null that removes a stored key.""" + + @model_serializer + def _as_null(self) -> None: + return None + + +class LiteLLMParamsPatch(BaseModel): + api_base: str | Clear | None = None + max_input_tokens: int | Clear | None = None + input_cost_per_token: float | Clear | None = None + output_cost_per_token: float | Clear | None = None + + +class ModelInfoPatch(BaseModel): + mode: ModelMode | Clear | None = None + max_input_tokens: int | Clear | None = None + + +class ModelPatchBody(BaseModel): + """PATCH /model/{model_id}/update body, JSON Merge Patch over the stored deployment: + a field left None is dropped from the body and unchanged, a field set to `Clear()` + is sent as null and removed, a field with a value is set.""" + + model_config = ConfigDict(protected_namespaces=()) + litellm_params: LiteLLMParamsPatch | None = None + model_info: ModelInfoPatch | None = None + + class ModelListEntry(BaseModel): id: str diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 1bac5116a9d..fa1b06fe7ed 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -10,10 +10,10 @@ from __future__ import annotations import time import warnings -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass -from functools import reduce from datetime import datetime +from functools import reduce from types import MappingProxyType from typing import Final @@ -62,6 +62,7 @@ from models import ( ModelMode, ModelNewBody, ModelNewResponse, + ModelPatchBody, ModelsListParams, ModelsListResponse, ModelUpdateBody, @@ -72,6 +73,7 @@ from models import ( SpendLogsPage, SpendLogsPageParams, SpendLogsParams, + StoredDeployment, ToolsetCreateBody, ToolsetRow, ToolsetUpdateBody, @@ -132,6 +134,103 @@ class NotServableOn: last_result: Result[ModelsListResponse] | None +type BodyReader[R: BaseModel] = Callable[[float], Result[R]] + + +@dataclass(frozen=True, slots=True) +class BodyNotConverged[R: BaseModel]: + """The deadline passed without a read the predicate accepted; `last_result` is the + final read, so the caller can tell a body that never matched from a read that + failed.""" + + last_result: Result[R] | None + + +@dataclass(frozen=True, slots=True) +class BodyConverged[R: BaseModel]: + """Every replica answered a body the predicate accepted; `bodies` is the last read + per replica.""" + + bodies: Mapping[str, R] + + +@dataclass(frozen=True, slots=True) +class BodyNeverConvergedOn[R: BaseModel]: + """`BodyNotConverged` labeled with the replica whose reads never satisfied the predicate.""" + + replica: str + last_result: Result[R] | None + + +def await_body_converged[R: BaseModel]( + read: BodyReader[R], + *, + predicate: Callable[[R], bool], + timeout: float, + interval: float, + request_timeout: float, + now: Callable[[], float], + sleep: Callable[[float], None], +) -> Success[R] | BodyNotConverged[R]: + """Poll `read` until it answers a body `predicate` accepts, or `timeout` passes. + + Each read's request timeout is clamped to the remaining budget, and the sleep + between reads to the time left, so the last read before the deadline is never + skipped. Clock and sleep are injected.""" + deadline: Final = now() + timeout + + def reads() -> Iterator[Result[R]]: + while (remaining := deadline - now()) > 0: + yield read(min(request_timeout, remaining)) + sleep(min(interval, max(deadline - now(), 0.0))) + + def attempts() -> Iterator[Success[R] | BodyNotConverged[R]]: + for result in reads(): + if isinstance(result, Success) and predicate(result.data): + yield result + return + yield BodyNotConverged(last_result=result) + + initial: Final[Success[R] | BodyNotConverged[R]] = BodyNotConverged(last_result=None) + return reduce(lambda _previous, result: result, attempts(), initial) + + +def await_body_converged_everywhere[R: BaseModel]( + readers: Mapping[str, BodyReader[R]], + *, + predicate: Callable[[R], bool], + timeout: float, + interval: float, + request_timeout: float, + now: Callable[[], float], + sleep: Callable[[float], None], +) -> BodyConverged[R] | BodyNeverConvergedOn[R]: + """`await_body_converged` against every replica in turn, each with the full budget, so a + write counts as landed only once every replica serves it.""" + def read_replica( + outcome: BodyConverged[R] | BodyNeverConvergedOn[R], + item: tuple[str, BodyReader[R]], + ) -> BodyConverged[R] | BodyNeverConvergedOn[R]: + if isinstance(outcome, BodyNeverConvergedOn): + return outcome + replica, read = item + match await_body_converged( + read, + predicate=predicate, + timeout=timeout, + interval=interval, + request_timeout=request_timeout, + now=now, + sleep=sleep, + ): + case Success(data=data): + return BodyConverged(bodies=MappingProxyType({**outcome.bodies, replica: data})) + case BodyNotConverged(last_result=last_result): + return BodyNeverConvergedOn(replica=replica, last_result=last_result) + initial: Final[BodyConverged[R] | BodyNeverConvergedOn[R]] = BodyConverged(bodies=MappingProxyType({})) + return reduce(read_replica, readers.items(), initial) + + def await_servable( list_models: Callable[[float], Result[ModelsListResponse]], *, @@ -658,6 +757,102 @@ class ProxyClient: ) ) + def patch_model(self, model_id: str, body: ModelPatchBody) -> StoredDeployment: + """JSON Merge Patch the deployment `model_id` via PATCH /model/{model_id}/update: + a field the body omits is unchanged, one sent as null is removed from the stored + row, one sent with a value is set. See ModelPatchBody for how a null is sent. + Returns the row as stored after the write.""" + return unwrap( + self.transport.patch( + f"/model/{model_id}/update", + headers=self.transport.master, + json=body, + response_type=StoredDeployment, + ) + ) + + def read_model_back_everywhere[R: BaseModel]( + self, path: str, response_type: type[R], *, predicate: Callable[[R], bool] + ) -> Mapping[str, R]: + """GET `path` on every replica until each answers a body `predicate` accepts, + polling to poll_timeout, and return the last body per replica. + + Fails naming the replica that never converged, so a write that reached one + gateway but not the others is caught instead of passing on whichever gateway + the balancer answered from. Falls back to the single proxy address when no + replica list is configured. + + `path` must be a data-plane route. The replicas are gateways, which serve only + the LLM surface, so a control-plane path answers on exactly one service and + 404s on every replica in a split deployment: asking each replica for one is + never the question the caller means. Read those through `self.transport` + instead, which routes them to the control plane.""" + if is_control_plane_path(path): + raise AssertionError( + f"read_model_back_everywhere({path!r}) asks every data-plane replica for a control-plane route. " + "The replicas are gateways and do not serve it; poll a data-plane path such as /v1/models " + "here, and read the control plane through the shared transport." + ) + readers: Final = { + url: self._body_reader(transport, path, response_type) + for url, transport in self._read_back_replicas().items() + } + outcome: Final = await_body_converged_everywhere( + readers, + predicate=predicate, + timeout=self.poll_timeout, + interval=self.poll_interval, + request_timeout=REQUEST_TIMEOUT, + now=time.monotonic, + sleep=time.sleep, + ) + match outcome: + case BodyConverged(bodies=bodies): + return bodies + case BodyNeverConvergedOn(replica=replica, last_result=last_result): + raise AssertionError( + f"GET {path} on {replica} never answered the expected body within " + f"{self.poll_timeout}s; last read: {last_result}" + ) + + def read_model_back[R: BaseModel](self, path: str, response_type: type[R], *, predicate: Callable[[R], bool]) -> R: + """GET `path` through the shared transport until the body satisfies `predicate`, + polling to poll_timeout, and return that body. + + The counterpart to `read_model_back_everywhere` for a control-plane route such as + /model/info: the stored row lives in one database behind one control plane, so + there is a single answer to converge on rather than one per gateway.""" + outcome: Final = await_body_converged_everywhere( + {CONTROL_PLANE_BASE_URL: self._body_reader(self.transport, path, response_type)}, + predicate=predicate, + timeout=self.poll_timeout, + interval=self.poll_interval, + request_timeout=REQUEST_TIMEOUT, + now=time.monotonic, + sleep=time.sleep, + ) + match outcome: + case BodyConverged(bodies=bodies): + return bodies[CONTROL_PLANE_BASE_URL] + case BodyNeverConvergedOn(last_result=last_result): + raise AssertionError( + f"GET {path} never answered the expected body within " + f"{self.poll_timeout}s; last read: {last_result}" + ) + + def _read_back_replicas(self) -> Mapping[str, Transport]: + return self.replicas or MappingProxyType({CONTROL_PLANE_BASE_URL: self.transport}) + + @staticmethod + def _body_reader[R: BaseModel](transport: Transport, path: str, response_type: type[R]) -> BodyReader[R]: + return lambda timeout: transport.get( + path, + headers=transport.master, + params=NoBody(), + response_type=response_type, + timeout=timeout, + ) + def delete_model(self, model_id: str) -> None: result = self.transport.post( "/model/delete", diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py index 3b84a47e3cc..cbf7f5648d4 100644 --- a/tests/e2e/test_proxy_client.py +++ b/tests/e2e/test_proxy_client.py @@ -20,8 +20,12 @@ from typing import Final, cast import pytest from e2e_config import parse_replica_urls from e2e_http import Result, Success -from models import KeyInfo, KeyInfoResponse, ModelListEntry, ModelsListResponse +from models import KeyInfo, KeyInfoResponse, ModelInfoEntry, ModelInfoResponse, ModelListEntry, ModelsListResponse from proxy_client import ( + BodyReader, + BodyConverged, + BodyNeverConvergedOn, + await_body_converged_everywhere, ConvergeOutcome, Converged, EverywhereConverged, @@ -274,3 +278,54 @@ class TestReplicasFor: client: Final = ProxyClient(transport=_NO_TRANSPORTS, replicas={}, control_replicas={}) with pytest.raises(AssertionError, match="no replica is configured"): _ = client.replicas_for("/v1/models") + + +def _info(*model_names: str) -> Success[ModelInfoResponse]: + entries: Final = [ModelInfoEntry(model_name=model_name) for model_name in model_names] + return Success(status_code=200, data=ModelInfoResponse(data=entries)) + + +def _reader(results: Iterable[Success[ModelInfoResponse]]) -> BodyReader[ModelInfoResponse]: + it: Final = iter(results) + return lambda _timeout: next(it) + + +def _lists_model(body: ModelInfoResponse) -> bool: + return any(entry.model_name == MODEL for entry in body.data) + + +def _read_back( + readers: Mapping[str, BodyReader[ModelInfoResponse]], +) -> tuple[BodyConverged[ModelInfoResponse] | BodyNeverConvergedOn[ModelInfoResponse], FakeClock]: + clock: Final = FakeClock() + outcome: Final = await_body_converged_everywhere( + readers, + predicate=_lists_model, + timeout=TIMEOUT, + interval=INTERVAL, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + return outcome, clock + + +class TestAwaitBodyConvergedEverywhere: + def test_waits_for_the_lagging_replica_and_returns_every_body(self) -> None: + readers: Final = { + "gateway-1": _reader(repeat(_info(MODEL))), + "gateway-2": _reader(chain(repeat(_info(), 2), repeat(_info(MODEL)))), + } + outcome, clock = _read_back(readers) + assert outcome == BodyConverged(bodies={"gateway-1": _info(MODEL).data, "gateway-2": _info(MODEL).data}) + assert clock.elapsed == 2 * INTERVAL + + @pytest.mark.parametrize("lagging", ["gateway-1", "gateway-2"]) + def test_fails_naming_the_replica_that_never_converges(self, lagging: str) -> None: + readers: Final = { + "gateway-1": _reader(repeat(_info(MODEL))), + "gateway-2": _reader(repeat(_info(MODEL))), + } | {lagging: _reader(repeat(_info()))} + outcome, clock = _read_back(readers) + assert outcome == BodyNeverConvergedOn(replica=lagging, last_result=_info()) + assert clock.elapsed >= TIMEOUT diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index 44fdbaa3e41..804e073a4a0 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -306,6 +306,7 @@ CONTROL_PLANE_PREFIXES: tuple[str, ...] = ( "/config", "/guardrails", "/openapi.json", + "/public/", ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index c02f886fc31..1376727e296 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -3115,9 +3115,6 @@ class TestUpdateDBModelClearPricing: """Sending an explicit `null` for a pricing field must remove it from both `litellm_params` and `model_info` (SPECIAL_MODEL_INFO_PARAMS are mirrored between the two by Deployment.__init__). - - Restricted to SPECIAL_MODEL_INFO_PARAMS so non-pricing fields (e.g. team_id) - cannot be cleared via this path. """ def test_clear_input_cost_removes_from_both_blobs(self): @@ -3193,10 +3190,10 @@ class TestUpdateDBModelClearPricing: assert params["input_cost_per_token"] == 0.000001 assert params["output_cost_per_token"] == 0.000007 - def test_null_on_non_pricing_field_does_not_clear(self): - """Security guard: only SPECIAL_MODEL_INFO_PARAMS can be cleared via null. - Privileged or unrelated model_info fields (e.g. team_id) must be unaffected - by the null-clearing path so a team admin can't ungate a team-scoped model. + def test_null_on_one_field_leaves_other_fields_alone(self): + """A null clears only the key it names: pricing the patch never mentions and + the ownership key team_id stay put, so a team admin can't ungate a + team-scoped model through the clear path. """ from litellm.proxy.management_endpoints.model_management_endpoints import ( update_db_model, @@ -3217,8 +3214,6 @@ class TestUpdateDBModelClearPricing: model_info=ModelInfo(id="dep-pricing-1", team_id="team-keep-me"), ) - # Patch sends a null for api_base (non-SPECIAL field). Must NOT clear team_id - # or any other non-pricing field from the merged dict. result = update_db_model( db_model=db_model, updated_patch=updateDeployment( @@ -3394,6 +3389,171 @@ class TestUpdateDBModelClearPricing: assert info["cache_creation_input_token_cost"] == 0.000003 +_PROTECTED_MODEL_INFO_VALUES = { + "team_id": "team-keep-me", + "team_public_model_name": "team-facing-name", + "access_groups": ["group-a"], + "created_at": "2026-01-01T00:00:00+00:00", + "created_by": "creator", + "updated_at": "2026-01-02T00:00:00+00:00", + "updated_by": "updater", + "blocked": True, +} + + +def _build_db_model_with_pinned_model_info(): + """Deployment whose stored blobs pin non-pricing keys an earlier save wrote, next to a + pricing override, so a clear can be checked key by key.""" + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + return Deployment( + model_name="pinned-gpt-4o-mini", + litellm_params=LiteLLM_Params( + model="gpt-4o-mini", input_cost_per_token=0.000001, max_input_tokens=4096 + ), + model_info=ModelInfo( + id="dep-pinned-0", + max_input_tokens=4096, + mode="chat", + supports_vision=True, + **_PROTECTED_MODEL_INFO_VALUES, + ), + ) + + +class TestUpdateDBModelNullClearsAnyKey: + """JSON Merge Patch on PATCH /model/{id}/update: a key sent as null is removed from the + stored blob it was sent in, whatever the key, except the identity and ownership keys, + whose nulls are ignored.""" + + def test_model_info_nulls_remove_pinned_non_pricing_keys(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate( + {"model_info": {"max_input_tokens": None, "mode": None}} + ), + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info + assert "mode" not in info + assert info["supports_vision"] is True + assert info["input_cost_per_token"] == 0.000001 + + def test_litellm_params_null_removes_pinned_non_pricing_key(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate( + {"litellm_params": {"max_input_tokens": None}} + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in params + assert params["model"] == "gpt-4o-mini" + assert params["input_cost_per_token"] == 0.000001 + assert info["max_input_tokens"] == 4096 + + def test_omitted_key_is_untouched_by_a_null_elsewhere(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate( + {"model_info": {"mode": None, "supports_vision": False}} + ), + ) + + info = json.loads(result["model_info"]) + assert "mode" not in info + assert info["supports_vision"] is False + assert info["max_input_tokens"] == 4096 + + @pytest.mark.parametrize("field", sorted(_PROTECTED_MODEL_INFO_VALUES)) + def test_null_on_protected_key_is_ignored(self, field): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate({"model_info": {field: None}}), + ) + + info = json.loads(result["model_info"]) + assert info[field] == _PROTECTED_MODEL_INFO_VALUES[field] + assert info["max_input_tokens"] == 4096 + + def test_echoing_the_read_back_blob_preserves_every_stored_key(self): + """The Admin UI edit form submits the whole /model/info row back, and that read reports + every key the deployment never stored as an explicit null. Those nulls have to stay + no-ops: a write drops None before storing, so a null in the echoed blob always names a + key the stored row does not carry. + """ + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + db_model = _build_db_model_with_pinned_model_info() + echoed = { + "id": "dep-pinned-0", + "max_input_tokens": 4096, + "mode": "chat", + "supports_vision": True, + "input_cost_per_token": 0.000001, + "team_id": "team-keep-me", + "base_model": None, + "tier": None, + "max_output_tokens": None, + "supports_function_calling": None, + "cache_read_input_token_cost": None, + } + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment.model_validate({"model_info": echoed}), + ) + + info = json.loads(result["model_info"]) + assert info["max_input_tokens"] == 4096 + assert info["mode"] == "chat" + assert info["supports_vision"] is True + assert info["input_cost_per_token"] == 0.000001 + assert info["team_id"] == "team-keep-me" + for never_stored in ("base_model", "tier", "max_output_tokens", "supports_function_calling"): + assert never_stored not in info + + def test_null_on_pricing_key_still_clears_both_blobs(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate( + {"model_info": {"input_cost_per_token": None}} + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "input_cost_per_token" not in params + assert "input_cost_per_token" not in info + assert params["max_input_tokens"] == 4096 + assert info["max_input_tokens"] == 4096 + + class TestGetModelInfoWithIdBlocked: """`ProxyConfig.get_model_info_with_id` must propagate the DB-level `blocked` column into the in-memory `model_info` dict so the router filter can read it.""" diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 30b265905f3..d22ec60e61a 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -220,6 +220,198 @@ def test_should_store_full_pricing_under_deployment_model_id(): assert entry["output_cost_per_token"] == 0.0 +def test_should_drop_a_price_the_deployment_no_longer_carries(): + """Re-registering a deployment must replace its model_id entry, not merge onto it. + + A merge left the old rate in the cost map after an operator cleared the override, so + the deployment kept billing at a price its config no longer had. + """ + backend_model = "vertex_ai/gemini-2.5-flash" + model_id = "deployment-cleared-price" + original = {model_id: litellm.model_cost.get(model_id)} + + try: + Router._register_deployment_in_model_cost( + model_id=model_id, + model_info={"input_cost_per_token": 0.005, "output_cost_per_token": 0.01}, + model=backend_model, + custom_llm_provider="vertex_ai", + ) + assert litellm.model_cost[model_id]["input_cost_per_token"] == 0.005 + + Router._register_deployment_in_model_cost( + model_id=model_id, + model_info={"mode": "chat"}, + model=backend_model, + custom_llm_provider="vertex_ai", + ) + + entry = litellm.model_cost[model_id] + assert entry.get("input_cost_per_token") != 0.005, ( + "the cleared override survived re-registration, so the deployment still bills at it" + ) + assert entry.get("output_cost_per_token") != 0.01 + finally: + _restore_model_cost_entries(original) + + +def test_should_not_strip_a_builtin_entry_when_a_deployment_id_collides_with_it(): + """Deployments are keyed into the same cost map as the built-in catalog, so a deployment + whose id happens to name a real model must not evict that model's entry. + + Stripping it would take the pricing and capability flags every other deployment of that + model reads, process-wide, until the next price-map reload. Registering twice, because + the first registration is what would mark the entry as this deployment's own. + """ + colliding_id = "gpt-4o" + original = {colliding_id: litellm.model_cost.get(colliding_id)} + builtin_max_tokens = litellm.model_cost[colliding_id]["max_tokens"] + + try: + for _ in range(2): + Router._register_deployment_in_model_cost( + model_id=colliding_id, + model_info={"id": colliding_id, "db_model": True, "mode": "chat"}, + model="gpt-4o-mini", + custom_llm_provider="openai", + ) + + entry = litellm.model_cost[colliding_id] + assert entry["max_tokens"] == builtin_max_tokens, ( + "registering a deployment under a catalog model's name wiped that model's context window" + ) + assert entry["litellm_provider"] == "openai" + assert entry["supports_vision"] is True + finally: + _restore_model_cost_entries(original) + + +def test_should_drop_a_stale_price_even_when_the_deployment_declares_a_provider(): + """A deployment may carry `litellm_provider` in its own model_info, which must not be + read as "this is a catalog entry" and stop the stale price from being dropped.""" + model_id = "deployment-provider-tagged" + original = {model_id: litellm.model_cost.get(model_id)} + + try: + Router._register_deployment_in_model_cost( + model_id=model_id, + model_info={"id": model_id, "litellm_provider": "openai", "input_cost_per_token": 0.005}, + model="gpt-4o-mini", + custom_llm_provider="openai", + ) + assert litellm.model_cost[model_id]["input_cost_per_token"] == 0.005 + + Router._register_deployment_in_model_cost( + model_id=model_id, + model_info={"id": model_id, "litellm_provider": "openai", "mode": "chat"}, + model="gpt-4o-mini", + custom_llm_provider="openai", + ) + + assert litellm.model_cost[model_id].get("input_cost_per_token") != 0.005, ( + "a deployment that declares its provider kept billing at the price it no longer carries" + ) + finally: + _restore_model_cost_entries(original) + + +def test_should_give_a_cost_map_key_back_when_the_deployment_is_deleted(): + """Deleting a deployment releases its claim on the shared cost-map key. + + Held forever, a later catalog refresh that starts publishing a model under that same + name would be treated as the deleted deployment's own entry and evicted. + """ + from litellm.router import _DEPLOYMENT_COST_MAP_KEYS + + model_id = "deployment-to-delete" + original = {model_id: litellm.model_cost.get(model_id)} + router = Router( + model_list=[ + { + "model_name": "to-delete", + "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"id": model_id, "input_cost_per_token": 0.005}, + } + ] + ) + + try: + assert model_id in _DEPLOYMENT_COST_MAP_KEYS + + assert router.delete_deployment(id=model_id) is not None + + assert model_id not in _DEPLOYMENT_COST_MAP_KEYS, ( + "a deleted deployment kept its claim on the shared cost-map key" + ) + finally: + _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) + _restore_model_cost_entries(original) + + +def test_should_keep_the_cost_map_key_while_another_router_still_serves_it(): + """Two live routers can serve the same deployment id, and the claim is process-wide. + + Releasing it when only one of them drops the deployment would put the survivor back on + merging, so the price it just cleared would keep billing. + """ + from litellm.router import _DEPLOYMENT_COST_MAP_KEYS + + model_id = "deployment-served-twice" + original = {model_id: litellm.model_cost.get(model_id)} + entry = { + "model_name": "served-twice", + "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"id": model_id, "input_cost_per_token": 0.005}, + } + first = Router(model_list=[entry]) + second = Router(model_list=[entry]) + + try: + assert model_id in _DEPLOYMENT_COST_MAP_KEYS + + assert first.delete_deployment(id=model_id) is not None + + assert model_id in _DEPLOYMENT_COST_MAP_KEYS, ( + "the claim was released while another router still served the deployment" + ) + + assert second.delete_deployment(id=model_id) is not None + assert model_id not in _DEPLOYMENT_COST_MAP_KEYS + finally: + _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) + _restore_model_cost_entries(original) + + +def test_should_keep_the_cost_map_key_while_a_dynamically_built_router_serves_it(): + """A router built with no model_list still serves whatever add_deployment gives it, so it + counts when deciding whether the shared cost-map claim can be released.""" + from litellm.router import _DEPLOYMENT_COST_MAP_KEYS + + model_id = "deployment-added-dynamically" + original = {model_id: litellm.model_cost.get(model_id)} + entry = { + "model_name": "added-dynamically", + "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"id": model_id, "input_cost_per_token": 0.005}, + } + configured = Router(model_list=[entry]) + dynamic = Router() + dynamic.add_deployment(deployment=Deployment(**entry)) + + try: + assert configured.delete_deployment(id=model_id) is not None + + assert model_id in _DEPLOYMENT_COST_MAP_KEYS, ( + "the claim was released while a dynamically built router still served the deployment" + ) + + assert dynamic.delete_deployment(id=model_id) is not None + assert model_id not in _DEPLOYMENT_COST_MAP_KEYS + finally: + _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) + _restore_model_cost_entries(original) + + def test_should_preserve_builtin_pricing_regardless_of_deployment_order(): """ The built-in pricing should be preserved no matter which deployment diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 83b0d58f2b2..93aaf3ca58c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9064,8 +9064,9 @@ export interface paths { * Patch Model * @description PATCH Endpoint for partial model updates. * - * Only updates the fields specified in the request while preserving other existing values. - * Follows proper PATCH semantics by only modifying provided fields. + * JSON Merge Patch semantics over `litellm_params` and `model_info`: a key absent from the + * body is unchanged, a key sent as null is removed from the stored row, and a key sent with a + * value is set (identity and ownership keys such as `id` and `team_id` ignore a null). * * Args: * model_id: The ID of the model to update From 25a6c2a2749ba802514fb3a5736b540c21a1e630 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 06:17:59 +0000 Subject: [PATCH 178/411] fix(cli): skip remote model cost map fetch in lite CLI processes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/get_model_cost_map.py | 14 ++++- .../test_get_model_cost_map.py | 25 +++++++++ .../proxy/client/cli/test_global_options.py | 56 ++++++++++++++++++- 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index cdc4810ff04..7d3b67e58c8 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -2,6 +2,7 @@ Pulls the cost + context window + provider route for known models from https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json This can be disabled by setting the LITELLM_LOCAL_MODEL_COST_MAP environment variable to True. +The ``lite`` and ``litellm-proxy`` CLI entry points also use the bundled map without fetching. ``` export LITELLM_LOCAL_MODEL_COST_MAP=True @@ -13,11 +14,13 @@ import hashlib import json import os import random +import sys import time from collections.abc import Awaitable, Callable from dataclasses import dataclass, replace from datetime import datetime, timezone from importlib.resources import files +from pathlib import Path from typing import Final, Protocol import httpx @@ -33,6 +36,12 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" +_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) + + +def _is_cli_process() -> bool: + return Path(sys.argv[0]).stem in _CLI_ENTRYPOINT_NAMES + # Reserved top-level keys that are not model entries. They must be excluded # from the model-count integrity check so a real upstream shrink can't be masked. @@ -531,7 +540,8 @@ def get_model_cost_map( """ Public entry point — returns the model cost map dict. - 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only. + 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set or this is a ``lite`` / + ``litellm-proxy`` CLI process, uses the local backup only. 2. Otherwise fetches from ``url``, retrying transient HTTP errors (429/5xx/transport) with Retry-After-aware backoff, validates integrity, and falls back to the local backup on any failure. @@ -543,7 +553,7 @@ def get_model_cost_map( _cost_map_source_info.loaded_at = datetime.now(timezone.utc) # Note: can't use get_secret_bool here — this runs during litellm.__init__ # before litellm._key_management_settings is set. - if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process(): _cost_map_source_info.source = "local" _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 0495440c51c..d092b387259 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -6,6 +6,7 @@ count actual model entries, not reserved meta keys) and the extraction of the import json import os +import sys import pytest @@ -711,3 +712,27 @@ def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rej assert source["etag"] is None assert source["source_revision"] == _bundled_blob_id() assert source["source_revision"] != git_blob_id(shrunk_body) + + +@pytest.mark.parametrize( + ("argv0", "request_count"), + [ + ("/some/venv/bin/lite", 0), + ("/some/venv/bin/lite.exe", 0), + ("/some/venv/bin/python", 1), + ], +) +def test_boot_load_skips_remote_fetch_for_cli_processes(monkeypatch, argv0, request_count): + monkeypatch.setattr(sys, "argv", [argv0, "--version"]) + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) + client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) + + cost_map = get_model_cost_map(url=_URL, client=client) + + assert calls["count"] == request_count + assert cost_map + source = get_model_cost_map_source_info() + if request_count == 0: + assert source["source"] == "local" + else: + assert source["source"] == "remote" diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 0dd388919a5..02b1cdf4296 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -1,14 +1,16 @@ # stdlib imports import json import os +import shutil +import subprocess +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from unittest.mock import Mock, patch import pytest from click.testing import CliRunner - - import litellm.proxy.client.cli from litellm._version import version as litellm_version from litellm.proxy.client.cli import cli @@ -35,6 +37,56 @@ def test_cli_version_flag(cli_runner): assert "LiteLLM Proxy Server Version: 1.2.3" in result.output +def test_lite_version_does_not_fetch_model_cost_map(): + lite_path = shutil.which("lite") + if lite_path is None: + pytest.skip("lite executable is unavailable") + + requests = [] + + class _CostMapHandler(BaseHTTPRequestHandler): + def do_GET(self): + requests.append(self.path) + body = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), _CostMapHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + env = {key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"} + source_root = str(Path(__file__).resolve().parents[5]) + env["PYTHONPATH"] = os.pathsep.join(filter(None, (source_root, env.get("PYTHONPATH")))) + env.update( + { + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": "http://127.0.0.1:9", + } + ) + result = subprocess.run( + [lite_path, "--version"], + capture_output=True, + text=True, + timeout=120, + env=env, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=10) + + assert result.returncode == 0 + assert "LiteLLM Proxy CLI Version" in result.stdout + assert requests == [] + + def test_cli_source_is_ascii_only(): """Non-ASCII output (emoji, box-drawing chars) raises UnicodeEncodeError on legacy Windows consoles (cp1252), so the whole CLI package must stay ASCII-only.""" From ffae447649e173214a2081b4890410a11dfdf085 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 06:30:50 +0000 Subject: [PATCH 179/411] test(cli): type cost map bypass regression handlers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/test_get_model_cost_map.py | 4 +++- tests/test_litellm/proxy/client/cli/test_global_options.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index d092b387259..6ab3f9a21af 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -722,7 +722,9 @@ def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rej ("/some/venv/bin/python", 1), ], ) -def test_boot_load_skips_remote_fetch_for_cli_processes(monkeypatch, argv0, request_count): +def test_boot_load_skips_remote_fetch_for_cli_processes( + monkeypatch: pytest.MonkeyPatch, argv0: str, request_count: int +) -> None: monkeypatch.setattr(sys, "argv", [argv0, "--version"]) monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 02b1cdf4296..6de56573ffb 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -42,10 +42,10 @@ def test_lite_version_does_not_fetch_model_cost_map(): if lite_path is None: pytest.skip("lite executable is unavailable") - requests = [] + requests: list[str] = [] class _CostMapHandler(BaseHTTPRequestHandler): - def do_GET(self): + def do_GET(self) -> None: requests.append(self.path) body = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' self.send_response(200) @@ -54,7 +54,7 @@ def test_lite_version_does_not_fetch_model_cost_map(): self.end_headers() self.wfile.write(body) - def log_message(self, format, *args): + def log_message(self, format: str, *args: object) -> None: return server = ThreadingHTTPServer(("127.0.0.1", 0), _CostMapHandler) From ef3a3c16ae02bc6d83e14b09c437ef36081c0498 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:32:31 -0700 Subject: [PATCH 180/411] feat(guardrails): map each guardrail scan id to its guardrail, stage and provider (#40327) * feat(guardrails): map each guardrail scan id to its guardrail, stage and provider Adds the x-litellm-guardrail-scan-metadata response header, a JSON list of {guardrail, stage, provider, scan_id} entries, next to the existing comma-separated x-litellm-guardrail-scan-id header. Prisma AIRS records the execution stage for every scan and OpenAI Moderation now records its moderation id too. The new metadata key is internal: client-supplied values are stripped and it is exposed through the UI CORS allow list. Resolves LIT-6018 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(guardrails): cap the scan metadata response header at a configurable length Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(guardrails): hardcode the scan metadata header cap Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 2 + litellm/proxy/common_utils/callback_utils.py | 62 ++++++++- .../guardrail_hooks/openai/moderations.py | 10 +- .../panw_prisma_airs/panw_prisma_airs.py | 34 +++-- litellm/proxy/litellm_pre_call_utils.py | 2 + .../proxy/common_utils/test_callback_utils.py | 122 +++++++++++++++--- .../openai/test_moderations.py | 32 +++++ .../guardrail_hooks/test_panw_prisma_airs.py | 32 ++++- 8 files changed, 265 insertions(+), 31 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 6f2384c8c6c..108a914e9c1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -143,6 +143,7 @@ DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = float( os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3) ) MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH: Final = int(os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150)) +MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH: Final = 2048 DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS: Final = 2000 @@ -197,6 +198,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [ "x-litellm-adaptive-router-model", "x-litellm-applied-guardrails", "x-litellm-guardrail-scan-id", + "x-litellm-guardrail-scan-metadata", "x-litellm-cache-key", ] diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 770963a1f24..561a53409f4 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -1,10 +1,12 @@ import copy +import json import os from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass +from itertools import accumulate from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias -from typing_extensions import assert_never +from typing_extensions import ReadOnly, TypedDict, assert_never import litellm from litellm import get_secret @@ -12,6 +14,7 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import ( CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, + MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH, PRE_CALL_EXECUTED_GUARDRAILS_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, @@ -28,6 +31,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.types_utils.utils import get_instance_fn +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( StandardLoggingGuardrailInformation, StandardLoggingPayload, @@ -52,6 +56,15 @@ reset_color_code: Final = "\033[0m" TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY: Final = "_pillar_response_headers_trusted" GUARDRAIL_SCAN_IDS_METADATA_KEY: Final = "guardrail_scan_ids" +GUARDRAIL_SCAN_METADATA_METADATA_KEY: Final = "guardrail_scan_metadata" + + +class GuardrailScanMetadata(TypedDict): + guardrail: ReadOnly[str | None] + stage: ReadOnly[str] + provider: ReadOnly[str] + scan_id: ReadOnly[str] + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -450,6 +463,16 @@ def get_remaining_tokens_and_requests_from_request_data(data: dict) -> dict[str, return headers +def _serialize_scan_metadata_header(entries: Iterable[object], *, max_length: int) -> str | None: + """Compact JSON list of scan metadata entries, dropping trailing entries so the header fits in max_length.""" + encoded: Final = tuple(json.dumps(entry, separators=(",", ":")) for entry in entries) + lengths: Final = tuple(accumulate(len(item) + 1 for item in encoded)) + kept: Final = sum(1 for length in lengths if length + 1 <= max_length) + if kept == 0: + return None + return f"[{','.join(encoded[:kept])}]" + + def get_logging_caching_headers(request_data: dict) -> dict | None: _metadata: Final[dict] = {} metadata_bucket: Final = request_data.get("metadata") @@ -468,6 +491,15 @@ def get_logging_caching_headers(request_data: dict) -> dict | None: if scan_ids: headers["x-litellm-guardrail-scan-id"] = ",".join(scan_ids) + scan_metadata: Final = _metadata.get(GUARDRAIL_SCAN_METADATA_METADATA_KEY) + scan_metadata_header: Final = ( + _serialize_scan_metadata_header(scan_metadata, max_length=MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH) + if isinstance(scan_metadata, (list, tuple)) + else None + ) + if scan_metadata_header: + headers["x-litellm-guardrail-scan-metadata"] = scan_metadata_header + if "applied_policies" in _metadata: headers["x-litellm-applied-policies"] = ",".join(_metadata["applied_policies"]) @@ -501,6 +533,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset( "applied_policies", "applied_guardrails", GUARDRAIL_SCAN_IDS_METADATA_KEY, + GUARDRAIL_SCAN_METADATA_METADATA_KEY, "policy_sources", "guardrails", "guardrail_config", @@ -565,21 +598,40 @@ def add_guardrail_to_applied_guardrails_header(request_data: dict, guardrail_nam _metadata["applied_guardrails"] = [guardrail_name] -def add_guardrail_scan_id(request_data: dict, scan_id: str | None) -> None: +def add_guardrail_scan_id( + request_data: dict[str, object], + scan_id: str | None, + *, + guardrail_name: str | None, + provider: str, + stage: GuardrailEventHooks, +) -> None: """ - Record a provider scan id so it can be surfaced to the caller. + Record a provider scan id, keyed to the guardrail execution that produced it, so it can be surfaced to the caller. Guardrails only return scan details to the client when they block, so allowed requests carry no - audit trail. Ids recorded here become the x-litellm-guardrail-scan-id response header. + audit trail. Ids recorded here become the x-litellm-guardrail-scan-id response header, and the + (guardrail, stage, provider, scan_id) entries become the x-litellm-guardrail-scan-metadata header. """ if not scan_id: return _, _metadata = get_or_create_metadata_bucket(request_data) existing: Final = _metadata.get(GUARDRAIL_SCAN_IDS_METADATA_KEY) - scan_ids: Final = tuple(existing) if isinstance(existing, (list, tuple)) else () + scan_ids: Final[tuple[object, ...]] = tuple(existing) if isinstance(existing, (list, tuple)) else () if scan_id not in scan_ids: _metadata[GUARDRAIL_SCAN_IDS_METADATA_KEY] = (*scan_ids, scan_id) + entry: Final[GuardrailScanMetadata] = { + "guardrail": guardrail_name, + "stage": stage.value, + "provider": provider, + "scan_id": scan_id, + } + existing_entries: Final = _metadata.get(GUARDRAIL_SCAN_METADATA_METADATA_KEY) + entries: Final[tuple[object, ...]] = tuple(existing_entries) if isinstance(existing_entries, (list, tuple)) else () + if entry not in entries: + _metadata[GUARDRAIL_SCAN_METADATA_METADATA_KEY] = (*entries, entry) + def add_policy_to_applied_policies_header(request_data: dict, policy_name: str | None): """ diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 10683550f85..c22d35509c1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -17,7 +17,8 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.types.guardrails import GuardrailEventHooks +from litellm.proxy.common_utils.callback_utils import add_guardrail_scan_id +from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations from litellm.types.utils import ( GenericGuardrailAPIInputs, GuardrailStatus, @@ -218,6 +219,13 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): metadata: Final = request_data.get("metadata") or {} request_data["metadata"] = metadata metadata["_openai_moderation_response"] = moderation_response.model_dump() + add_guardrail_scan_id( + request_data=request_data, + scan_id=moderation_response.id, + guardrail_name=self.guardrail_name, + provider=SupportedGuardrailIntegrations.OPENAI_MODERATION.value, + stage=GuardrailEventHooks.post_call if input_type == "response" else GuardrailEventHooks.pre_call, + ) # Check if content is flagged and raise exception if needed self._check_moderation_result(moderation_response) diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index b73d3adb99e..3bc0dfabefc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -721,10 +721,18 @@ class PanwPrismaAirsHandler(CustomGuardrail): } } - def _record_scan_id(self, request_data: dict[str, object], scan_result: Mapping[str, object]) -> None: + def _record_scan_id( + self, request_data: dict[str, object], scan_result: Mapping[str, object], stage: GuardrailEventHooks + ) -> None: """Surface the AIRS scan id on the response, so allowed calls are auditable too.""" scan_id: Final = scan_result.get("scan_id") - add_guardrail_scan_id(request_data=request_data, scan_id=str(scan_id) if scan_id else None) + add_guardrail_scan_id( + request_data=request_data, + scan_id=str(scan_id) if scan_id else None, + guardrail_name=self.guardrail_name, + provider=self._PROVIDER_NAME, + stage=stage, + ) def _handle_api_error_with_logging( self, @@ -948,7 +956,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): event_type=GuardrailEventHooks.post_call, ) add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) - self._record_scan_id(request_data, scan_result) + self._record_scan_id(request_data, scan_result, GuardrailEventHooks.post_call) def _check_and_mark_scanned(self, data: dict, scan_type: str) -> bool: """ @@ -1078,7 +1086,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): duration=(end_time - start_time).total_seconds(), event_type=GuardrailEventHooks.pre_call, ) - self._record_scan_id(data, scan_result) + self._record_scan_id(data, scan_result, GuardrailEventHooks.pre_call) action: Final = scan_result.get("action", "block") category: Final = scan_result.get("category", "unknown") @@ -1199,7 +1207,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): duration=(end_time - start_time).total_seconds(), event_type=GuardrailEventHooks.post_call, ) - self._record_scan_id(data, scan_result) + self._record_scan_id(data, scan_result, GuardrailEventHooks.post_call) action: Final = scan_result.get("action", "block") category: Final = scan_result.get("category", "unknown") @@ -1401,7 +1409,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): duration=(end_time - start_time).total_seconds(), event_type=GuardrailEventHooks.post_call, ) - self._record_scan_id(request_data, scan_result) + self._record_scan_id(request_data, scan_result, GuardrailEventHooks.post_call) # Add guardrail to applied guardrails header for observability add_guardrail_to_applied_guardrails_header( @@ -1475,7 +1483,11 @@ class PanwPrismaAirsHandler(CustomGuardrail): ) continue - self._record_scan_id(request_data, scan_result) + self._record_scan_id( + request_data, + scan_result, + GuardrailEventHooks.post_call if is_response else GuardrailEventHooks.pre_call, + ) action = scan_result.get("action", "block") masked_args = self._masked_tool_call_arguments( @@ -1829,7 +1841,11 @@ class PanwPrismaAirsHandler(CustomGuardrail): new_texts.append(text) continue - self._record_scan_id(request_data, scan_result) + self._record_scan_id( + request_data, + scan_result, + GuardrailEventHooks.post_call if is_response else GuardrailEventHooks.pre_call, + ) action = scan_result.get("action", "block") masked_text = self._get_masked_text(scan_result, is_response=is_response) @@ -1901,7 +1917,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): ) # If we reach here, fallback_on_error="allow" else: - self._record_scan_id(request_data, mcp_scan_result) + self._record_scan_id(request_data, mcp_scan_result, GuardrailEventHooks.pre_call) action = mcp_scan_result.get("action", "block") masked_text = self._get_masked_text(mcp_scan_result, is_response=False) if action == "allow": diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 924f84be5f4..3250ae5cca9 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -235,6 +235,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "applied_policies", "policy_sources", "guardrail_scan_ids", + "guardrail_scan_metadata", "routing_decision", GATEWAY_INJECTED_CACHE_METADATA_KEY, "pillar_response_headers", @@ -291,6 +292,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "applied_policies", "policy_sources", "guardrail_scan_ids", + "guardrail_scan_metadata", "routing_decision", GATEWAY_INJECTED_CACHE_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index 66f77db6da9..ecb2375d495 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -1,30 +1,33 @@ import copy +import json import sys from types import ModuleType, SimpleNamespace +from typing import Final +from unittest.mock import patch import pytest - +import litellm +from litellm.caching.caching import DualCache +from litellm.constants import MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( + _serialize_scan_metadata_header, add_guardrail_scan_id, add_policy_to_applied_policies_header, decrypt_callback_vars, encrypt_callback_vars, get_logging_caching_headers, - initialize_callbacks_on_proxy, get_remaining_tokens_and_requests_from_request_data, + initialize_callbacks_on_proxy, normalize_callback_names, + process_callback, sanitize_openai_provider_metadata, strip_callback_config, ) -import litellm -from litellm.caching.caching import DualCache -from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging - -from unittest.mock import patch -from litellm.proxy.common_utils.callback_utils import process_callback +from litellm.types.guardrails import GuardrailEventHooks def test_get_remaining_tokens_and_requests_from_request_data(): @@ -189,20 +192,109 @@ def test_get_logging_caching_headers_merges_metadata_and_litellm_metadata(): assert headers["x-litellm-policy-sources"] == "global-baseline=team_default" +def _record( + request_data: dict[str, object], + scan_id: str | None, + guardrail_name: str = "airs", + provider: str = "panw_prisma_airs", + stage: GuardrailEventHooks = GuardrailEventHooks.pre_call, +) -> None: + add_guardrail_scan_id( + request_data=request_data, scan_id=scan_id, guardrail_name=guardrail_name, provider=provider, stage=stage + ) + + def test_add_guardrail_scan_id_dedupes_and_becomes_response_header(): request_data = {"litellm_metadata": {}} - add_guardrail_scan_id(request_data=request_data, scan_id="scan-1") - add_guardrail_scan_id(request_data=request_data, scan_id="scan-1") - add_guardrail_scan_id(request_data=request_data, scan_id="scan-2") - add_guardrail_scan_id(request_data=request_data, scan_id=None) + _record(request_data, "scan-1") + _record(request_data, "scan-1") + _record(request_data, "scan-2") + _record(request_data, None) assert request_data["litellm_metadata"]["guardrail_scan_ids"] == ("scan-1", "scan-2") assert get_logging_caching_headers(request_data)["x-litellm-guardrail-scan-id"] == "scan-1,scan-2" -def test_get_logging_caching_headers_omits_scan_id_header_without_scans(): - assert "x-litellm-guardrail-scan-id" not in get_logging_caching_headers({"litellm_metadata": {}}) +def test_scan_metadata_header_maps_each_id_to_its_guardrail_stage_and_provider(): + request_data: Final[dict[str, object]] = {"litellm_metadata": {}} + + _record( + request_data, "scan-1", guardrail_name="airs", provider="panw_prisma_airs", stage=GuardrailEventHooks.pre_call + ) + _record( + request_data, "mod-1", guardrail_name="mod", provider="openai_moderation", stage=GuardrailEventHooks.pre_call + ) + _record( + request_data, "scan-2", guardrail_name="airs", provider="panw_prisma_airs", stage=GuardrailEventHooks.post_call + ) + _record( + request_data, "scan-2", guardrail_name="airs", provider="panw_prisma_airs", stage=GuardrailEventHooks.post_call + ) + _record(request_data, None, guardrail_name="mod", provider="openai_moderation", stage=GuardrailEventHooks.post_call) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + assert headers["x-litellm-guardrail-scan-id"] == "scan-1,mod-1,scan-2" + assert json.loads(headers["x-litellm-guardrail-scan-metadata"]) == [ + {"guardrail": "airs", "stage": "pre_call", "provider": "panw_prisma_airs", "scan_id": "scan-1"}, + {"guardrail": "mod", "stage": "pre_call", "provider": "openai_moderation", "scan_id": "mod-1"}, + {"guardrail": "airs", "stage": "post_call", "provider": "panw_prisma_airs", "scan_id": "scan-2"}, + ] + + +def test_scan_metadata_keeps_same_id_reused_across_stages(): + request_data: Final[dict[str, object]] = {"metadata": {}} + + _record(request_data, "scan-1", stage=GuardrailEventHooks.pre_call) + _record(request_data, "scan-1", stage=GuardrailEventHooks.post_call) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + assert headers["x-litellm-guardrail-scan-id"] == "scan-1" + assert [entry["stage"] for entry in json.loads(headers["x-litellm-guardrail-scan-metadata"])] == [ + "pre_call", + "post_call", + ] + + +def test_scan_metadata_header_drops_trailing_entries_to_stay_within_length_limit(): + request_data: Final[dict[str, object]] = {"litellm_metadata": {}} + scan_ids: Final = tuple(f"0f9c4b7e-3d2a-4c1b-9e8f-{index:012d}" for index in range(40)) + for scan_id in scan_ids: + _record(request_data, scan_id, stage=GuardrailEventHooks.post_call) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + assert headers["x-litellm-guardrail-scan-id"] == ",".join(scan_ids) + header: Final = headers["x-litellm-guardrail-scan-metadata"] + assert len(header) <= MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH + kept: Final = json.loads(header) + assert 1 < len(kept) < len(scan_ids) + assert [entry["scan_id"] for entry in kept] == list(scan_ids[: len(kept)]) + + +def test_serialize_scan_metadata_header_keeps_exactly_the_entries_that_fit(): + entries: Final = ({"scan_id": "a"}, {"scan_id": "b"}, {"scan_id": "c"}) + two_entries: Final = '[{"scan_id":"a"},{"scan_id":"b"}]' + + assert _serialize_scan_metadata_header(entries, max_length=len(two_entries)) == two_entries + assert _serialize_scan_metadata_header(entries, max_length=len(two_entries) - 1) == '[{"scan_id":"a"}]' + assert _serialize_scan_metadata_header(entries, max_length=len(two_entries) + 1) == two_entries + assert _serialize_scan_metadata_header(entries, max_length=1000) == json.dumps(entries, separators=(",", ":")) + assert _serialize_scan_metadata_header(entries, max_length=5) is None + assert _serialize_scan_metadata_header((), max_length=1000) is None + + +def test_scan_metadata_is_an_internal_metadata_key(): + assert sanitize_openai_provider_metadata({"guardrail_scan_metadata": "x", "keep": "y"}) == {"keep": "y"} + + +def test_get_logging_caching_headers_omits_scan_headers_without_scans(): + headers: Final = get_logging_caching_headers({"litellm_metadata": {}}) + assert headers is not None + assert "x-litellm-guardrail-scan-id" not in headers + assert "x-litellm-guardrail-scan-metadata" not in headers def test_initialize_callbacks_on_proxy_instantiates_compression_interception( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py index 2b43720a126..615d06b0f42 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py @@ -3,14 +3,19 @@ Test OpenAI Moderation Guardrail """ +import json import os +from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.callback_utils import get_logging_caching_headers from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import ( OpenAIModerationGuardrail, ) @@ -989,3 +994,30 @@ async def test_openai_moderation_initialize_guardrail_forwards_streaming_flags() assert guardrail.streaming_sampling_rate == 2 finally: litellm.logging_callback_manager._reset_all_callbacks() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("input_type", "stage"), [("request", "pre_call"), ("response", "post_call")]) +async def test_openai_moderation_records_moderation_id_as_scan_metadata(input_type: str, stage: str): + """Each moderation call's id is exposed with the guardrail name, stage and provider that produced it.""" + payload: Final = { + "id": f"modr-{stage}", + "model": "omni-moderation-latest", + "results": [{"flagged": False, "categories": {}, "category_scores": {}, "category_applied_input_types": {}}], + } + http_client: Final = AsyncHTTPHandler() + http_client.client = httpx.AsyncClient(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=payload))) + + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): + guardrail: Final = OpenAIModerationGuardrail(guardrail_name="openai-mod") + guardrail.async_handler = http_client + request_data: Final[dict[str, object]] = {"metadata": {}} + + await guardrail.apply_guardrail(inputs={"texts": ["hello"]}, request_data=request_data, input_type=input_type) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + assert headers["x-litellm-guardrail-scan-id"] == f"modr-{stage}" + assert json.loads(headers["x-litellm-guardrail-scan-metadata"]) == [ + {"guardrail": "openai-mod", "stage": stage, "provider": "openai_moderation", "scan_id": f"modr-{stage}"} + ] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index 8f29ba66814..3d7c6e06d94 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -12,6 +12,7 @@ This test file follows LiteLLM's testing patterns and covers: import copy import json from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -5785,7 +5786,14 @@ class TestPanwAirsScanIdExposure: headers = get_logging_caching_headers(data) assert headers["x-litellm-guardrail-scan-id"] == "scan-abc-123" - assert "x-litellm-guardrail-scan-metadata" not in headers + assert json.loads(headers["x-litellm-guardrail-scan-metadata"]) == [ + { + "guardrail": handler.guardrail_name, + "stage": "pre_call", + "provider": "panw_prisma_airs", + "scan_id": "scan-abc-123", + } + ] @pytest.mark.asyncio async def test_request_and_response_scan_ids_are_both_exposed(self, user_api_key_dict): @@ -5809,6 +5817,26 @@ class TestPanwAirsScanIdExposure: headers = get_logging_caching_headers(data) assert headers["x-litellm-guardrail-scan-id"] == "scan-abc-123,scan-response-456" + assert [(e["stage"], e["scan_id"]) for e in json.loads(headers["x-litellm-guardrail-scan-metadata"])] == [ + ("pre_call", "scan-abc-123"), + ("post_call", "scan-response-456"), + ] + + @pytest.mark.asyncio + async def test_apply_guardrail_response_scan_is_tagged_post_call(self): + from litellm.proxy.common_utils.callback_utils import get_logging_caching_headers + + handler: Final = self._handler(self.ALLOW_SCAN_RESULT) + request_data: Final[dict[str, object]] = {"litellm_call_id": "test-call-id", "model": "gpt-4", "metadata": {}} + + await handler.apply_guardrail( + inputs={"texts": ["Hello world"]}, request_data=request_data, input_type="response" + ) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + entries: Final = json.loads(headers["x-litellm-guardrail-scan-metadata"]) + assert [(e["stage"], e["provider"]) for e in entries] == [("post_call", "panw_prisma_airs")] @pytest.mark.asyncio async def test_repeated_scan_id_is_not_duplicated(self, user_api_key_dict): @@ -5850,6 +5878,8 @@ class TestPanwAirsScanIdExposure: assert "guardrail_scan_ids" in _UNTRUSTED_METADATA_CONTROL_FIELDS assert "guardrail_scan_ids" in _UNTRUSTED_ROOT_CONTROL_FIELDS + assert "guardrail_scan_metadata" in _UNTRUSTED_METADATA_CONTROL_FIELDS + assert "guardrail_scan_metadata" in _UNTRUSTED_ROOT_CONTROL_FIELDS class TestPanwAirsBlockedErrorDetailPassthrough: """Regression tests for the full AIRS scan response on blocks. From 0c0a1dd76f31b56059508dd674fee1764c83828f Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 06:34:36 +0000 Subject: [PATCH 181/411] test(cli): avoid mutable request tracking Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/client/cli/test_global_options.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 6de56573ffb..9ed9f91ab9f 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -37,16 +37,17 @@ def test_cli_version_flag(cli_runner): assert "LiteLLM Proxy Server Version: 1.2.3" in result.output -def test_lite_version_does_not_fetch_model_cost_map(): +def test_lite_version_does_not_fetch_model_cost_map(tmp_path: Path) -> None: lite_path = shutil.which("lite") if lite_path is None: pytest.skip("lite executable is unavailable") - requests: list[str] = [] + request_log: Path = tmp_path / "requests.log" class _CostMapHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: - requests.append(self.path) + with request_log.open("a", encoding="utf-8") as log_file: + log_file.write(f"{self.path}\n") body = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' self.send_response(200) self.send_header("Content-Type", "application/json") @@ -84,7 +85,8 @@ def test_lite_version_does_not_fetch_model_cost_map(): assert result.returncode == 0 assert "LiteLLM Proxy CLI Version" in result.stdout - assert requests == [] + request_count: int = request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 + assert request_count == 0 def test_cli_source_is_ascii_only(): From 9d90a544916c6f38397862dd18b6a4c12de98827 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 9 Sep 2026 00:06:26 -0700 Subject: [PATCH 182/411] revert(model-management): roll back #40047 This reverts commit e8e3172d7d70558929f32f057ebe4c7471c8c352 Restore the previous model update and router cost registration behavior while pricing compatibility is investigated --- .../model_management_endpoints.py | 70 +--- litellm/router.py | 26 +- tests/e2e/coverage_registry/mgmt.yaml | 2 - .../management/test_model_lifecycle_e2e.py | 365 ------------------ tests/e2e/models.py | 74 +--- tests/e2e/proxy_client.py | 199 +--------- tests/e2e/test_proxy_client.py | 57 +-- tests/e2e/transport.py | 1 - .../test_model_management_endpoints.py | 178 +-------- .../test_router_model_cost_isolation.py | 192 --------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 +- 11 files changed, 39 insertions(+), 1130 deletions(-) delete mode 100644 tests/e2e/management/test_model_lifecycle_e2e.py diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 742b9d9817f..0f19e9ce149 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -119,7 +119,6 @@ from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, Deployment, GenericLiteLLMParams, - LiteLLM_Params, ModelInfo, updateDeployment, ) @@ -729,44 +728,6 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -_OWNERSHIP_FIELDS: Final = frozenset( - { - "db_model", - "team_id", - "team_public_model_name", - "access_groups", - "created_at", - "created_by", - "updated_at", - "updated_by", - "blocked", - } -) - -_STORED_REQUIRED_FIELDS: Final = frozenset( - name for model in (LiteLLM_Params, ModelInfo) for name, field in model.model_fields.items() if field.is_required() -) - -_NULL_CLEAR_IGNORED_FIELDS: Final = _OWNERSHIP_FIELDS | _STORED_REQUIRED_FIELDS | frozenset(PTU_MODEL_INFO_FIELDS) - - -def _explicitly_cleared_fields(patch: BaseModel | None) -> frozenset[str]: - """The keys a patch sends as an explicit null, which update_db_model removes from the - stored blob (JSON Merge Patch). Ownership keys are left alone, as are the keys the stored - models require, since clearing one writes a row no reload can rebuild through - LiteLLM_Params / ModelInfo. The PTU keys are handled by _explicitly_cleared_ptu_fields, - whose clear is gated on the feature flag. Applied after both blobs merge, so a model_info blob - the UI echoes back cannot resurrect a pricing key the litellm_params patch clears. - """ - if patch is None: - return frozenset() - return frozenset( - field - for field in patch.model_fields_set - if field not in _NULL_CLEAR_IGNORED_FIELDS and getattr(patch, field) is None - ) - - def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) @@ -787,15 +748,25 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr if updated_patch.model_info: merged_model_info.update(updated_patch.model_info.model_dump(exclude_none=True)) - for field in _explicitly_cleared_fields(updated_patch.litellm_params): - merged_litellm_params.pop(field, None) - if field in SPECIAL_MODEL_INFO_PARAMS: - merged_model_info.pop(field, None) - for field in _explicitly_cleared_fields(updated_patch.model_info): - merged_model_info.pop(field, None) - if field in SPECIAL_MODEL_INFO_PARAMS: - merged_litellm_params.pop(field, None) + # Honor explicit-null clears LAST, after both merges, so a model_info blob the UI + # passes through (which today re-sends the OLD pricing on every save) cannot + # silently undo a litellm_params clear via .update(). + # + # Restricted to SPECIAL_MODEL_INFO_PARAMS (input/output cost per token/character + # and cache read/write costs) so this path cannot be used to null out privileged + # model_info fields like team_id or access groups. SPECIAL_MODEL_INFO_PARAMS are + # mirrored between litellm_params and model_info by Deployment.__init__, so the + # clear propagates to both blobs. + if updated_patch.litellm_params: + for field in updated_patch.litellm_params.model_fields_set: + if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.litellm_params, field) is None: + merged_litellm_params.pop(field, None) + merged_model_info.pop(field, None) if updated_patch.model_info: + for field in updated_patch.model_info.model_fields_set: + if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.model_info, field) is None: + merged_model_info.pop(field, None) + merged_litellm_params.pop(field, None) for field in _explicitly_cleared_ptu_fields(updated_patch.model_info): merged_model_info.pop(field, None) @@ -845,9 +816,8 @@ async def patch_model( """ PATCH Endpoint for partial model updates. - JSON Merge Patch semantics over `litellm_params` and `model_info`: a key absent from the - body is unchanged, a key sent as null is removed from the stored row, and a key sent with a - value is set (identity and ownership keys such as `id` and `team_id` ignore a null). + Only updates the fields specified in the request while preserving other existing values. + Follows proper PATCH semantics by only modifying provided fields. Args: model_id: The ID of the model to update diff --git a/litellm/router.py b/litellm/router.py index 1252d7e7487..934a4ac86a9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -628,15 +628,6 @@ RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( RETRY_BREADCRUMB_LIMIT: Final = 4 -# Cost-map keys created by _register_deployment_in_model_cost, which shares one flat -# namespace with the built-in model catalog. Only a key it created may be evicted, or a -# deployment whose id names a real model would strip that model's pricing and -# capabilities for every other deployment of it. delete_deployment gives a key back once no -# live router still serves that id, so a later catalog refresh that starts serving the name -# is not treated as a deployment's own. -_DEPLOYMENT_COST_MAP_KEYS: Final[set[str]] = set() # mutable-ok: ownership of shared cost-map keys - - class FallbackAwareStreamWrapper(CustomStreamWrapper): """Base for the Router's chat-completion stream wrappers, which are built around the attempt the Router picked first and have to repoint themselves when a fallback takes over.""" @@ -9770,7 +9761,6 @@ class Router: """ idx: Final = len(self.model_list) self.model_list.append(model) - _live_routers.add(self) # mutable-ok: track dynamic routers without extending their lifetimes self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() @@ -9939,12 +9929,7 @@ class Router: """Write a deployment's metadata into ``litellm.model_cost``. Runs when a deployment is added and again after a price data reload, so - the entries a refresh rebuilds are the ones a fresh boot would produce. An - entry this function created is replaced rather than merged, so a price cleared - from the deployment does not linger from an earlier registration and keep - billing at the old rate. An entry it did not create is left to merge, because - a deployment id that collides with a catalog model name shares that model's - entry with every other deployment of it. + the entries a refresh rebuilds are the ones a fresh boot would produce. Nothing is recorded for replay: a refresh walks the live routers instead, so a deleted, repointed or never-added deployment, and a discarded router, drop out of the rebuild on their own. @@ -9961,10 +9946,6 @@ class Router: } if model_id is not None: - if model_id in _DEPLOYMENT_COST_MAP_KEYS: - litellm.model_cost.pop(model_id, None) # mutable-ok: remove cleared prices from the shared entry - elif model_id not in litellm.model_cost: - _DEPLOYMENT_COST_MAP_KEYS.add(model_id) # mutable-ok: retain shared ownership across reloads litellm.register_model( model_cost={model_id: model_info}, persist_across_reloads=False, @@ -10061,11 +10042,6 @@ class Router: _budget_limiter: Final = self._get_router_deployment_budget_limiter() if _budget_limiter is not None: _budget_limiter.unregister_deployment_budget(model_id=id) - if not any( - router is not self and id in router.model_id_to_deployment_index_map - for router in tuple(_live_routers) - ): - _DEPLOYMENT_COST_MAP_KEYS.discard(id) # mutable-ok: the last owning router released this key try: self._unregister_pre_routing_strategy_for_deployment( deployment=item if isinstance(item, Deployment) else Deployment(**item) diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 83f1711a245..c8d7037d2fd 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -76,8 +76,6 @@ - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} - {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven} -- {id: mgmt.model.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "model_management_endpoints.py:731", rationale: "A partial PATCH changes only the keys it names; every other stored key reads back byte-for-byte, and the new rate reaches billing"} -- {id: mgmt.model.update.clear_persists, module: mgmt, tier: P0, surface: api, assertions: [clear_persists], source: "model_management_endpoints.py:731", fail_before_fix: proven, rationale: "An explicit null on PATCH removes the key from the stored row and billing falls back to the cost map; before the fix only mirrored pricing keys could be cleared"} - {id: mgmt.mcp_server.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1577", rationale: "Every field of an admin-created MCP server reads back verbatim, by id and in the list, on every replica"} - {id: mgmt.mcp_server.list.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1112", rationale: "The MCP page grid lists a created server with the same field values its detail view reports"} - {id: mgmt.mcp_server.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "mcp_management_endpoints.py:2665", rationale: "A dashboard edit of one field leaves the others intact and is visible on every replica after one save; edits that took several saves to stick were a customer defect"} diff --git a/tests/e2e/management/test_model_lifecycle_e2e.py b/tests/e2e/management/test_model_lifecycle_e2e.py deleted file mode 100644 index 99044efeb46..00000000000 --- a/tests/e2e/management/test_model_lifecycle_e2e.py +++ /dev/null @@ -1,365 +0,0 @@ -"""Live e2e: the lifecycle of a DB-stored deployment through the model management -routes, read back on every gateway replica. - -Each test registers its own gpt-4o-mini mock deployment through /model/new (deleted -on teardown) with non-default pricing, context window, mode, and api_base pinned, then -walks the lifecycle up to the step it proves: the create reads back field for field, -a partial PATCH changes only the key it names, an explicit null on PATCH removes the -key from the stored row (JSON Merge Patch), a call after the price clear is billed at -the cost map's rate rather than the cleared override, and a delete removes the -deployment from /model/info and makes the model name unknown to /chat/completions. - -The stored row is read back from /model/info, a control-plane route with one answer -behind it. What every gateway must agree on is which models it serves, so the create -and delete steps poll /v1/models on every URL in PROXY_REPLICA_URLS through -ProxyClient.read_model_back_everywhere, failing by name on the gateway that never converged. -""" - -from __future__ import annotations - -import math -import time -from collections.abc import Callable -from dataclasses import dataclass -from typing import Final - -import pytest -from pydantic import BaseModel - -from e2e_config import unique_marker -from e2e_http import unwrap -from lifecycle import ResourceManager -from management_client import ManagementClient -from models import ( - ChatBody, - ChatMessage, - Clear, - LiteLLMParamsBody, - LiteLLMParamsPatch, - ModelInfoBody, - ModelInfoEntry, - ModelInfoResponse, - ModelNewBody, - ModelPatchBody, - ModelsListResponse, - SpendLogRow, -) - -pytestmark = pytest.mark.e2e - -BACKEND_MODEL: Final = "gpt-4o-mini" -PINNED_API_BASE: Final = "https://pinned.example.invalid/v1" -PINNED_MAX_INPUT_TOKENS: Final = 4096 -PINNED_INPUT_RATE: Final = 1e-05 -UPDATED_INPUT_RATE: Final = 2e-05 -PINNED_OUTPUT_RATE: Final = 3e-05 - -# A PATCH lands on the control plane, and each gateway picks it up on its own config -# reload, so the first call after the write can still be billed at the old rate. There -# is no price on the gateway's data-plane surface to poll, so the billing steps drive -# calls until the new rate shows up in the spend row and let the deadline be what fails. -BILLING_CONVERGENCE_TIMEOUT: Final = 90.0 -BILLING_CONVERGENCE_INTERVAL: Final = 5.0 - - -@dataclass(frozen=True, slots=True) -class Registered: - model_name: str - model_id: str - - -class _ErrorDetail(BaseModel): - message: str - - -class _ErrorEnvelope(BaseModel): - error: _ErrorDetail - - -def _register(client: ManagementClient, resources: ResourceManager) -> Registered: - """Register a mock gpt-4o-mini deployment with every field under test pinned to a - non-default value, deleted on teardown. max_input_tokens is pinned in - litellm_params only: a value in model_info is copied into the shared cost-map - entry for the backend model, which would leak into every other gpt-4o-mini - deployment on the proxy.""" - model_name: Final = f"e2e-lifecycle-{unique_marker()}" - model_id: Final = client.proxy.register_model( - ModelNewBody( - model_name=model_name, - litellm_params=LiteLLMParamsBody( - model=BACKEND_MODEL, - mock_response="ok", - api_base=PINNED_API_BASE, - input_cost_per_token=PINNED_INPUT_RATE, - output_cost_per_token=PINNED_OUTPUT_RATE, - max_input_tokens=PINNED_MAX_INPUT_TOKENS, - ), - model_info=ModelInfoBody(mode="chat"), - ) - ) - resources.defer(lambda: client.proxy.delete_model(model_id)) - return Registered(model_name=model_name, model_id=model_id) - - -def _entry(body: ModelInfoResponse, model_name: str) -> ModelInfoEntry | None: - return next((entry for entry in body.data if entry.model_name == model_name), None) - - -def _stored_entry( - client: ManagementClient, - model_name: str, - *, - converged: Callable[[ModelInfoEntry], bool], -) -> ModelInfoEntry: - """The stored /model/info row for `model_name`, once it satisfies `converged`. - - /model/info is a control-plane route: the gateways named in PROXY_REPLICA_URLS - serve the LLM surface only, so the stored row has one answer, not one per - gateway. What every gateway must agree on is which models it serves, and - `_assert_served_everywhere` / `_assert_absent_everywhere` poll /v1/models for - that.""" - - def has_converged(body: ModelInfoResponse) -> bool: - entry: Final = _entry(body, model_name) - return entry is not None and converged(entry) - - body: Final = client.proxy.read_model_back("/model/info", ModelInfoResponse, predicate=has_converged) - entry: Final = _entry(body, model_name) - assert entry is not None, f"/model/info stopped listing {model_name!r} between the poll and the read" - return entry - - -def _serves(body: ModelsListResponse, model_name: str) -> bool: - return any(entry.id == model_name for entry in body.data) - - -def _assert_served_everywhere(client: ManagementClient, model_name: str) -> None: - _ = client.proxy.read_model_back_everywhere( - "/v1/models", ModelsListResponse, predicate=lambda body: _serves(body, model_name) - ) - - -def _assert_absent_everywhere(client: ManagementClient, model_name: str) -> None: - _ = client.proxy.read_model_back_everywhere( - "/v1/models", ModelsListResponse, predicate=lambda body: not _serves(body, model_name) - ) - - -def _assert_untouched_keys_as_created(entry: ModelInfoEntry, replica: str) -> None: - """The keys no later step names read back byte-for-byte as /model/new wrote them.""" - params: Final = entry.litellm_params - assert params.model == BACKEND_MODEL, f"{replica}: litellm_params.model {params.model!r} != {BACKEND_MODEL!r}" - assert params.api_base == PINNED_API_BASE, f"{replica}: api_base {params.api_base!r} != {PINNED_API_BASE!r}" - assert params.output_cost_per_token == PINNED_OUTPUT_RATE, ( - f"{replica}: output_cost_per_token {params.output_cost_per_token} != {PINNED_OUTPUT_RATE}" - ) - assert entry.model_info.mode == "chat", f"{replica}: model_info.mode {entry.model_info.mode!r} != 'chat'" - - -def _approx_equal(actual: float, expected: float) -> bool: - return math.isclose(actual, expected, rel_tol=1e-2, abs_tol=1e-9) - - -def _priced(rows: list[SpendLogRow]) -> bool: - return any(row.metadata and row.metadata.cost_breakdown and row.metadata.cost_breakdown.input_cost for row in rows) - - -def _billed_input_cost(client: ManagementClient, model_name: str, key: str) -> tuple[int, float]: - """Drive one chat completion through `model_name` and return the prompt tokens and - input cost its spend row recorded, so a test can assert the rate the gateway actually - billed rather than only the rate it stored.""" - chat: Final = unwrap( - client.proxy.chat( - key, - ChatBody( - model=model_name, - messages=[ChatMessage(role="user", content=f"reply with one word {unique_marker()}")], - max_tokens=16, - ), - ) - ) - assert chat.id is not None, f"chat completion carried no id to find its spend row by: {chat}" - - rows: Final = client.proxy.poll_logs_for_request_id(chat.id, predicate=_priced) - row: Final = next((row for row in rows if row.request_id == chat.id), None) - assert row is not None and row.metadata and row.metadata.cost_breakdown, ( - f"no priced spend row for request {chat.id} before the deadline: {rows}" - ) - prompt_tokens: Final = row.prompt_tokens or 0 - input_cost: Final = row.metadata.cost_breakdown.input_cost - assert prompt_tokens > 0 and input_cost is not None, f"spend row logged no prompt tokens or input cost: {row}" - return prompt_tokens, input_cost - - -def _await_billed_input_cost( - client: ManagementClient, model_name: str, key: str, *, expected_rate: float -) -> tuple[int, float]: - """Drive calls through `model_name` until one is billed at `expected_rate`, and - return the prompt tokens and input cost of the last spend row either way. - - Only the deadline ends the wait unsatisfied: a rate that never reaches the gateway - comes back as the stale cost for the caller to assert on, so the rate the caller - expects is still what decides the test.""" - deadline: Final = time.monotonic() + BILLING_CONVERGENCE_TIMEOUT - while True: - prompt_tokens, input_cost = _billed_input_cost(client, model_name, key) - if _approx_equal(input_cost, prompt_tokens * expected_rate) or time.monotonic() >= deadline: - return prompt_tokens, input_cost - time.sleep(BILLING_CONVERGENCE_INTERVAL) - - -class TestModelLifecycle: - @pytest.mark.covers("mgmt.model.add.persists") - def test_create_reads_back_every_field_and_serves_on_every_replica( - self, client: ManagementClient, resources: ResourceManager - ) -> None: - registered = _register(client, resources) - - entry = _stored_entry(client, registered.model_name, converged=lambda _entry: True) - stored = "/model/info" - - _assert_untouched_keys_as_created(entry, stored) - assert entry.litellm_params.input_cost_per_token == PINNED_INPUT_RATE, ( - f"{stored}: input_cost_per_token {entry.litellm_params.input_cost_per_token} != {PINNED_INPUT_RATE}" - ) - assert entry.litellm_params.max_input_tokens == PINNED_MAX_INPUT_TOKENS, ( - f"{stored}: max_input_tokens {entry.litellm_params.max_input_tokens} != {PINNED_MAX_INPUT_TOKENS}" - ) - assert entry.model_info.id == registered.model_id, ( - f"{stored}: model_info.id {entry.model_info.id!r} != {registered.model_id!r}" - ) - - _assert_served_everywhere(client, registered.model_name) - - @pytest.mark.covers("mgmt.model.update.preserves_unrelated_fields") - def test_partial_update_changes_only_the_named_key( - self, client: ManagementClient, resources: ResourceManager, scoped_key: str - ) -> None: - registered = _register(client, resources) - - stored = client.proxy.patch_model( - registered.model_id, - ModelPatchBody(litellm_params=LiteLLMParamsPatch(input_cost_per_token=UPDATED_INPUT_RATE)), - ) - assert stored.litellm_params.input_cost_per_token == UPDATED_INPUT_RATE, ( - f"PATCH response stores input_cost_per_token {stored.litellm_params.input_cost_per_token}, " - f"sent {UPDATED_INPUT_RATE}" - ) - - entry = _stored_entry( - client, - registered.model_name, - converged=lambda entry: entry.litellm_params.input_cost_per_token == UPDATED_INPUT_RATE, - ) - stored = "/model/info" - - _assert_untouched_keys_as_created(entry, stored) - assert entry.litellm_params.max_input_tokens == PINNED_MAX_INPUT_TOKENS, ( - f"{stored}: max_input_tokens {entry.litellm_params.max_input_tokens} != {PINNED_MAX_INPUT_TOKENS}" - ) - assert entry.model_info.input_cost_per_token == UPDATED_INPUT_RATE, ( - f"{stored}: model_info.input_cost_per_token {entry.model_info.input_cost_per_token} " - f"did not mirror the updated {UPDATED_INPUT_RATE}" - ) - - prompt_tokens, input_cost = _await_billed_input_cost( - client, registered.model_name, scoped_key, expected_rate=UPDATED_INPUT_RATE - ) - assert _approx_equal(input_cost, prompt_tokens * UPDATED_INPUT_RATE), ( - f"input_cost {input_cost} != {prompt_tokens} tokens * updated rate {UPDATED_INPUT_RATE} " - f"= {prompt_tokens * UPDATED_INPUT_RATE}; the partial update did not reach billing" - ) - - @pytest.mark.covers("mgmt.model.update.clear_persists") - def test_explicit_null_removes_the_key_from_the_stored_row( - self, client: ManagementClient, resources: ResourceManager - ) -> None: - registered = _register(client, resources) - - stored = client.proxy.patch_model( - registered.model_id, - ModelPatchBody(litellm_params=LiteLLMParamsPatch(max_input_tokens=Clear(), input_cost_per_token=Clear())), - ) - stored_params = stored.litellm_params.model_fields_set - assert "max_input_tokens" not in stored_params, ( - f"stored litellm_params still carries max_input_tokens " - f"{stored.litellm_params.max_input_tokens} after an explicit null" - ) - assert "input_cost_per_token" not in stored_params, ( - f"stored litellm_params still carries input_cost_per_token " - f"{stored.litellm_params.input_cost_per_token} after an explicit null" - ) - assert "max_input_tokens" not in stored.model_info.model_fields_set, ( - f"stored model_info carries max_input_tokens {stored.model_info.max_input_tokens} after the clear" - ) - assert "input_cost_per_token" not in stored.model_info.model_fields_set, ( - f"stored model_info still mirrors input_cost_per_token {stored.model_info.input_cost_per_token}" - ) - - cost_map_input_rate = client.proxy.model_cost_map()[BACKEND_MODEL].input_cost_per_token - assert cost_map_input_rate is not None, f"cost map has no input rate for {BACKEND_MODEL}" - entry = _stored_entry( - client, - registered.model_name, - converged=lambda entry: "max_input_tokens" not in entry.litellm_params.model_fields_set, - ) - stored = "/model/info" - - _assert_untouched_keys_as_created(entry, stored) - served = entry.litellm_params.model_fields_set - assert "max_input_tokens" not in served, ( - f"{stored}: litellm_params still serves max_input_tokens {entry.litellm_params.max_input_tokens}" - ) - assert "input_cost_per_token" not in served, ( - f"{stored}: litellm_params still serves input_cost_per_token {entry.litellm_params.input_cost_per_token}" - ) - assert entry.model_info.input_cost_per_token == cost_map_input_rate, ( - f"{stored}: model_info.input_cost_per_token {entry.model_info.input_cost_per_token} is not the " - f"cost map's {cost_map_input_rate}; the cleared override {PINNED_INPUT_RATE} still resolves" - ) - - @pytest.mark.covers("mgmt.model.update.clear_persists") - def test_cleared_price_is_billed_at_the_cost_map_rate( - self, client: ManagementClient, resources: ResourceManager, scoped_key: str - ) -> None: - registered = _register(client, resources) - _ = client.proxy.patch_model( - registered.model_id, - ModelPatchBody(litellm_params=LiteLLMParamsPatch(max_input_tokens=Clear(), input_cost_per_token=Clear())), - ) - _ = _stored_entry( - client, - registered.model_name, - converged=lambda entry: "input_cost_per_token" not in entry.litellm_params.model_fields_set, - ) - cost_map_input_rate = client.proxy.model_cost_map()[BACKEND_MODEL].input_cost_per_token - assert cost_map_input_rate is not None, f"cost map has no input rate for {BACKEND_MODEL}" - - prompt_tokens, input_cost = _await_billed_input_cost( - client, registered.model_name, scoped_key, expected_rate=cost_map_input_rate - ) - - assert _approx_equal(input_cost, prompt_tokens * cost_map_input_rate), ( - f"input_cost {input_cost} != {prompt_tokens} tokens * cost map rate {cost_map_input_rate} " - f"= {prompt_tokens * cost_map_input_rate}" - ) - assert not _approx_equal(input_cost, prompt_tokens * PINNED_INPUT_RATE), ( - f"input_cost {input_cost} is still billed at the cleared override {PINNED_INPUT_RATE}" - ) - - @pytest.mark.covers("mgmt.model.delete.persists") - def test_delete_removes_the_deployment_everywhere( - self, client: ManagementClient, resources: ResourceManager, scoped_key: str - ) -> None: - registered = _register(client, resources) - _ = _stored_entry(client, registered.model_name, converged=lambda _entry: True) - - client.delete_model_strict(registered.model_id) - - _assert_absent_everywhere(client, registered.model_name) - refused = client.chat_status(scoped_key, registered.model_name, "hi this is a test") - assert refused.status_code == 400, ( - f"chat against the deleted model must be rejected 400, got {refused.status_code}: {refused.body[:300]}" - ) - envelope = _ErrorEnvelope.model_validate_json(refused.body) - assert envelope.error.message, f"400 body must carry an error message: {refused.body[:300]}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 8db37bd25a5..62810e6cfd9 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -655,11 +655,6 @@ class OcrResponse(BaseModel): # ---------- spend logs ---------- -class CostBreakdown(BaseModel): - input_cost: float | None = None - output_cost: float | None = None - - class GuardrailEntityMatch(BaseModel): entity_type: str score: float @@ -677,7 +672,6 @@ class GuardrailRunRecord(BaseModel): class SpendLogMetadata(BaseModel): - cost_breakdown: CostBreakdown | None = None applied_guardrails: list[str] | None = None guardrail_information: list[GuardrailRunRecord] | None = None @@ -821,42 +815,15 @@ class CustomPricing(BaseModel): return prompt_tokens * self.input_cost_per_token + completion_tokens * self.output_cost_per_token -class DeploymentParams(CustomPricing): - """The litellm_params half of a /model/info row: the stored deployment as written, - credentials scrubbed. Unlike model_info it is never back-filled from the cost map, - so a key the store dropped is absent here (check `model_fields_set`).""" - - model: str | None = None - api_base: str | None = None - max_input_tokens: int | None = None - - -class DeploymentModelInfo(CustomPricing): - id: str | None = None - max_input_tokens: int | None = None - - class ModelInfoEntry(BaseModel): """One /model/info row. `litellm_params` is the configured deployment (carries any custom-pricing override); `model_info` is the price the proxy resolved for - it - the override merged over the cost-map defaults, so a key cleared from the - stored blob reads as the cost-map default here.""" + it - the override merged over the cost-map defaults.""" model_config = ConfigDict(protected_namespaces=()) model_name: str - litellm_params: DeploymentParams = DeploymentParams() - model_info: DeploymentModelInfo = DeploymentModelInfo() - - -class StoredDeployment(BaseModel): - """PATCH /model/{model_id}/update answers with the row as stored: both blobs raw, - nothing back-filled, so a cleared key is absent from `model_fields_set` of the - blob it was cleared from.""" - - model_config = ConfigDict(protected_namespaces=()) - model_name: str - litellm_params: DeploymentParams - model_info: DeploymentModelInfo + litellm_params: CustomPricing = CustomPricing() + model_info: CustomPricing = CustomPricing() class ModelInfoResponse(BaseModel): @@ -957,10 +924,9 @@ class LiteLLMParamsBody(BaseModel): timeout: float | None = None tpm: int | None = None weight: int | None = None - max_input_tokens: int | None = None -ModelMode = Literal["chat", "batch", "realtime", "image_generation"] +ModelMode = Literal["batch", "realtime", "image_generation"] class ModelInfoBody(BaseModel): @@ -970,7 +936,6 @@ class ModelInfoBody(BaseModel): # constraint when a prior run's teardown had not removed the row. id: str | None = None mode: ModelMode | None = None - max_input_tokens: int | None = None access_groups: list[str] | None = None team_id: str | None = None allowed_fails_policy: dict[str, int] | None = None @@ -999,37 +964,6 @@ class ModelUpdateBody(BaseModel): model_info: ModelInfoBody -class Clear(BaseModel): - """Serializes to JSON null. The transport dumps every body with exclude_none, so a - field set to this is how a patch carries the explicit null that removes a stored key.""" - - @model_serializer - def _as_null(self) -> None: - return None - - -class LiteLLMParamsPatch(BaseModel): - api_base: str | Clear | None = None - max_input_tokens: int | Clear | None = None - input_cost_per_token: float | Clear | None = None - output_cost_per_token: float | Clear | None = None - - -class ModelInfoPatch(BaseModel): - mode: ModelMode | Clear | None = None - max_input_tokens: int | Clear | None = None - - -class ModelPatchBody(BaseModel): - """PATCH /model/{model_id}/update body, JSON Merge Patch over the stored deployment: - a field left None is dropped from the body and unchanged, a field set to `Clear()` - is sent as null and removed, a field with a value is set.""" - - model_config = ConfigDict(protected_namespaces=()) - litellm_params: LiteLLMParamsPatch | None = None - model_info: ModelInfoPatch | None = None - - class ModelListEntry(BaseModel): id: str diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index fa1b06fe7ed..1bac5116a9d 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -10,10 +10,10 @@ from __future__ import annotations import time import warnings -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass -from datetime import datetime from functools import reduce +from datetime import datetime from types import MappingProxyType from typing import Final @@ -62,7 +62,6 @@ from models import ( ModelMode, ModelNewBody, ModelNewResponse, - ModelPatchBody, ModelsListParams, ModelsListResponse, ModelUpdateBody, @@ -73,7 +72,6 @@ from models import ( SpendLogsPage, SpendLogsPageParams, SpendLogsParams, - StoredDeployment, ToolsetCreateBody, ToolsetRow, ToolsetUpdateBody, @@ -134,103 +132,6 @@ class NotServableOn: last_result: Result[ModelsListResponse] | None -type BodyReader[R: BaseModel] = Callable[[float], Result[R]] - - -@dataclass(frozen=True, slots=True) -class BodyNotConverged[R: BaseModel]: - """The deadline passed without a read the predicate accepted; `last_result` is the - final read, so the caller can tell a body that never matched from a read that - failed.""" - - last_result: Result[R] | None - - -@dataclass(frozen=True, slots=True) -class BodyConverged[R: BaseModel]: - """Every replica answered a body the predicate accepted; `bodies` is the last read - per replica.""" - - bodies: Mapping[str, R] - - -@dataclass(frozen=True, slots=True) -class BodyNeverConvergedOn[R: BaseModel]: - """`BodyNotConverged` labeled with the replica whose reads never satisfied the predicate.""" - - replica: str - last_result: Result[R] | None - - -def await_body_converged[R: BaseModel]( - read: BodyReader[R], - *, - predicate: Callable[[R], bool], - timeout: float, - interval: float, - request_timeout: float, - now: Callable[[], float], - sleep: Callable[[float], None], -) -> Success[R] | BodyNotConverged[R]: - """Poll `read` until it answers a body `predicate` accepts, or `timeout` passes. - - Each read's request timeout is clamped to the remaining budget, and the sleep - between reads to the time left, so the last read before the deadline is never - skipped. Clock and sleep are injected.""" - deadline: Final = now() + timeout - - def reads() -> Iterator[Result[R]]: - while (remaining := deadline - now()) > 0: - yield read(min(request_timeout, remaining)) - sleep(min(interval, max(deadline - now(), 0.0))) - - def attempts() -> Iterator[Success[R] | BodyNotConverged[R]]: - for result in reads(): - if isinstance(result, Success) and predicate(result.data): - yield result - return - yield BodyNotConverged(last_result=result) - - initial: Final[Success[R] | BodyNotConverged[R]] = BodyNotConverged(last_result=None) - return reduce(lambda _previous, result: result, attempts(), initial) - - -def await_body_converged_everywhere[R: BaseModel]( - readers: Mapping[str, BodyReader[R]], - *, - predicate: Callable[[R], bool], - timeout: float, - interval: float, - request_timeout: float, - now: Callable[[], float], - sleep: Callable[[float], None], -) -> BodyConverged[R] | BodyNeverConvergedOn[R]: - """`await_body_converged` against every replica in turn, each with the full budget, so a - write counts as landed only once every replica serves it.""" - def read_replica( - outcome: BodyConverged[R] | BodyNeverConvergedOn[R], - item: tuple[str, BodyReader[R]], - ) -> BodyConverged[R] | BodyNeverConvergedOn[R]: - if isinstance(outcome, BodyNeverConvergedOn): - return outcome - replica, read = item - match await_body_converged( - read, - predicate=predicate, - timeout=timeout, - interval=interval, - request_timeout=request_timeout, - now=now, - sleep=sleep, - ): - case Success(data=data): - return BodyConverged(bodies=MappingProxyType({**outcome.bodies, replica: data})) - case BodyNotConverged(last_result=last_result): - return BodyNeverConvergedOn(replica=replica, last_result=last_result) - initial: Final[BodyConverged[R] | BodyNeverConvergedOn[R]] = BodyConverged(bodies=MappingProxyType({})) - return reduce(read_replica, readers.items(), initial) - - def await_servable( list_models: Callable[[float], Result[ModelsListResponse]], *, @@ -757,102 +658,6 @@ class ProxyClient: ) ) - def patch_model(self, model_id: str, body: ModelPatchBody) -> StoredDeployment: - """JSON Merge Patch the deployment `model_id` via PATCH /model/{model_id}/update: - a field the body omits is unchanged, one sent as null is removed from the stored - row, one sent with a value is set. See ModelPatchBody for how a null is sent. - Returns the row as stored after the write.""" - return unwrap( - self.transport.patch( - f"/model/{model_id}/update", - headers=self.transport.master, - json=body, - response_type=StoredDeployment, - ) - ) - - def read_model_back_everywhere[R: BaseModel]( - self, path: str, response_type: type[R], *, predicate: Callable[[R], bool] - ) -> Mapping[str, R]: - """GET `path` on every replica until each answers a body `predicate` accepts, - polling to poll_timeout, and return the last body per replica. - - Fails naming the replica that never converged, so a write that reached one - gateway but not the others is caught instead of passing on whichever gateway - the balancer answered from. Falls back to the single proxy address when no - replica list is configured. - - `path` must be a data-plane route. The replicas are gateways, which serve only - the LLM surface, so a control-plane path answers on exactly one service and - 404s on every replica in a split deployment: asking each replica for one is - never the question the caller means. Read those through `self.transport` - instead, which routes them to the control plane.""" - if is_control_plane_path(path): - raise AssertionError( - f"read_model_back_everywhere({path!r}) asks every data-plane replica for a control-plane route. " - "The replicas are gateways and do not serve it; poll a data-plane path such as /v1/models " - "here, and read the control plane through the shared transport." - ) - readers: Final = { - url: self._body_reader(transport, path, response_type) - for url, transport in self._read_back_replicas().items() - } - outcome: Final = await_body_converged_everywhere( - readers, - predicate=predicate, - timeout=self.poll_timeout, - interval=self.poll_interval, - request_timeout=REQUEST_TIMEOUT, - now=time.monotonic, - sleep=time.sleep, - ) - match outcome: - case BodyConverged(bodies=bodies): - return bodies - case BodyNeverConvergedOn(replica=replica, last_result=last_result): - raise AssertionError( - f"GET {path} on {replica} never answered the expected body within " - f"{self.poll_timeout}s; last read: {last_result}" - ) - - def read_model_back[R: BaseModel](self, path: str, response_type: type[R], *, predicate: Callable[[R], bool]) -> R: - """GET `path` through the shared transport until the body satisfies `predicate`, - polling to poll_timeout, and return that body. - - The counterpart to `read_model_back_everywhere` for a control-plane route such as - /model/info: the stored row lives in one database behind one control plane, so - there is a single answer to converge on rather than one per gateway.""" - outcome: Final = await_body_converged_everywhere( - {CONTROL_PLANE_BASE_URL: self._body_reader(self.transport, path, response_type)}, - predicate=predicate, - timeout=self.poll_timeout, - interval=self.poll_interval, - request_timeout=REQUEST_TIMEOUT, - now=time.monotonic, - sleep=time.sleep, - ) - match outcome: - case BodyConverged(bodies=bodies): - return bodies[CONTROL_PLANE_BASE_URL] - case BodyNeverConvergedOn(last_result=last_result): - raise AssertionError( - f"GET {path} never answered the expected body within " - f"{self.poll_timeout}s; last read: {last_result}" - ) - - def _read_back_replicas(self) -> Mapping[str, Transport]: - return self.replicas or MappingProxyType({CONTROL_PLANE_BASE_URL: self.transport}) - - @staticmethod - def _body_reader[R: BaseModel](transport: Transport, path: str, response_type: type[R]) -> BodyReader[R]: - return lambda timeout: transport.get( - path, - headers=transport.master, - params=NoBody(), - response_type=response_type, - timeout=timeout, - ) - def delete_model(self, model_id: str) -> None: result = self.transport.post( "/model/delete", diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py index cbf7f5648d4..3b84a47e3cc 100644 --- a/tests/e2e/test_proxy_client.py +++ b/tests/e2e/test_proxy_client.py @@ -20,12 +20,8 @@ from typing import Final, cast import pytest from e2e_config import parse_replica_urls from e2e_http import Result, Success -from models import KeyInfo, KeyInfoResponse, ModelInfoEntry, ModelInfoResponse, ModelListEntry, ModelsListResponse +from models import KeyInfo, KeyInfoResponse, ModelListEntry, ModelsListResponse from proxy_client import ( - BodyReader, - BodyConverged, - BodyNeverConvergedOn, - await_body_converged_everywhere, ConvergeOutcome, Converged, EverywhereConverged, @@ -278,54 +274,3 @@ class TestReplicasFor: client: Final = ProxyClient(transport=_NO_TRANSPORTS, replicas={}, control_replicas={}) with pytest.raises(AssertionError, match="no replica is configured"): _ = client.replicas_for("/v1/models") - - -def _info(*model_names: str) -> Success[ModelInfoResponse]: - entries: Final = [ModelInfoEntry(model_name=model_name) for model_name in model_names] - return Success(status_code=200, data=ModelInfoResponse(data=entries)) - - -def _reader(results: Iterable[Success[ModelInfoResponse]]) -> BodyReader[ModelInfoResponse]: - it: Final = iter(results) - return lambda _timeout: next(it) - - -def _lists_model(body: ModelInfoResponse) -> bool: - return any(entry.model_name == MODEL for entry in body.data) - - -def _read_back( - readers: Mapping[str, BodyReader[ModelInfoResponse]], -) -> tuple[BodyConverged[ModelInfoResponse] | BodyNeverConvergedOn[ModelInfoResponse], FakeClock]: - clock: Final = FakeClock() - outcome: Final = await_body_converged_everywhere( - readers, - predicate=_lists_model, - timeout=TIMEOUT, - interval=INTERVAL, - request_timeout=5.0, - now=clock.now, - sleep=clock.sleep, - ) - return outcome, clock - - -class TestAwaitBodyConvergedEverywhere: - def test_waits_for_the_lagging_replica_and_returns_every_body(self) -> None: - readers: Final = { - "gateway-1": _reader(repeat(_info(MODEL))), - "gateway-2": _reader(chain(repeat(_info(), 2), repeat(_info(MODEL)))), - } - outcome, clock = _read_back(readers) - assert outcome == BodyConverged(bodies={"gateway-1": _info(MODEL).data, "gateway-2": _info(MODEL).data}) - assert clock.elapsed == 2 * INTERVAL - - @pytest.mark.parametrize("lagging", ["gateway-1", "gateway-2"]) - def test_fails_naming_the_replica_that_never_converges(self, lagging: str) -> None: - readers: Final = { - "gateway-1": _reader(repeat(_info(MODEL))), - "gateway-2": _reader(repeat(_info(MODEL))), - } | {lagging: _reader(repeat(_info()))} - outcome, clock = _read_back(readers) - assert outcome == BodyNeverConvergedOn(replica=lagging, last_result=_info()) - assert clock.elapsed >= TIMEOUT diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index 804e073a4a0..44fdbaa3e41 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -306,7 +306,6 @@ CONTROL_PLANE_PREFIXES: tuple[str, ...] = ( "/config", "/guardrails", "/openapi.json", - "/public/", ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 1376727e296..c02f886fc31 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -3115,6 +3115,9 @@ class TestUpdateDBModelClearPricing: """Sending an explicit `null` for a pricing field must remove it from both `litellm_params` and `model_info` (SPECIAL_MODEL_INFO_PARAMS are mirrored between the two by Deployment.__init__). + + Restricted to SPECIAL_MODEL_INFO_PARAMS so non-pricing fields (e.g. team_id) + cannot be cleared via this path. """ def test_clear_input_cost_removes_from_both_blobs(self): @@ -3190,10 +3193,10 @@ class TestUpdateDBModelClearPricing: assert params["input_cost_per_token"] == 0.000001 assert params["output_cost_per_token"] == 0.000007 - def test_null_on_one_field_leaves_other_fields_alone(self): - """A null clears only the key it names: pricing the patch never mentions and - the ownership key team_id stay put, so a team admin can't ungate a - team-scoped model through the clear path. + def test_null_on_non_pricing_field_does_not_clear(self): + """Security guard: only SPECIAL_MODEL_INFO_PARAMS can be cleared via null. + Privileged or unrelated model_info fields (e.g. team_id) must be unaffected + by the null-clearing path so a team admin can't ungate a team-scoped model. """ from litellm.proxy.management_endpoints.model_management_endpoints import ( update_db_model, @@ -3214,6 +3217,8 @@ class TestUpdateDBModelClearPricing: model_info=ModelInfo(id="dep-pricing-1", team_id="team-keep-me"), ) + # Patch sends a null for api_base (non-SPECIAL field). Must NOT clear team_id + # or any other non-pricing field from the merged dict. result = update_db_model( db_model=db_model, updated_patch=updateDeployment( @@ -3389,171 +3394,6 @@ class TestUpdateDBModelClearPricing: assert info["cache_creation_input_token_cost"] == 0.000003 -_PROTECTED_MODEL_INFO_VALUES = { - "team_id": "team-keep-me", - "team_public_model_name": "team-facing-name", - "access_groups": ["group-a"], - "created_at": "2026-01-01T00:00:00+00:00", - "created_by": "creator", - "updated_at": "2026-01-02T00:00:00+00:00", - "updated_by": "updater", - "blocked": True, -} - - -def _build_db_model_with_pinned_model_info(): - """Deployment whose stored blobs pin non-pricing keys an earlier save wrote, next to a - pricing override, so a clear can be checked key by key.""" - from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo - - return Deployment( - model_name="pinned-gpt-4o-mini", - litellm_params=LiteLLM_Params( - model="gpt-4o-mini", input_cost_per_token=0.000001, max_input_tokens=4096 - ), - model_info=ModelInfo( - id="dep-pinned-0", - max_input_tokens=4096, - mode="chat", - supports_vision=True, - **_PROTECTED_MODEL_INFO_VALUES, - ), - ) - - -class TestUpdateDBModelNullClearsAnyKey: - """JSON Merge Patch on PATCH /model/{id}/update: a key sent as null is removed from the - stored blob it was sent in, whatever the key, except the identity and ownership keys, - whose nulls are ignored.""" - - def test_model_info_nulls_remove_pinned_non_pricing_keys(self): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate( - {"model_info": {"max_input_tokens": None, "mode": None}} - ), - ) - - info = json.loads(result["model_info"]) - assert "max_input_tokens" not in info - assert "mode" not in info - assert info["supports_vision"] is True - assert info["input_cost_per_token"] == 0.000001 - - def test_litellm_params_null_removes_pinned_non_pricing_key(self): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate( - {"litellm_params": {"max_input_tokens": None}} - ), - ) - - params = json.loads(result["litellm_params"]) - info = json.loads(result["model_info"]) - assert "max_input_tokens" not in params - assert params["model"] == "gpt-4o-mini" - assert params["input_cost_per_token"] == 0.000001 - assert info["max_input_tokens"] == 4096 - - def test_omitted_key_is_untouched_by_a_null_elsewhere(self): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate( - {"model_info": {"mode": None, "supports_vision": False}} - ), - ) - - info = json.loads(result["model_info"]) - assert "mode" not in info - assert info["supports_vision"] is False - assert info["max_input_tokens"] == 4096 - - @pytest.mark.parametrize("field", sorted(_PROTECTED_MODEL_INFO_VALUES)) - def test_null_on_protected_key_is_ignored(self, field): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate({"model_info": {field: None}}), - ) - - info = json.loads(result["model_info"]) - assert info[field] == _PROTECTED_MODEL_INFO_VALUES[field] - assert info["max_input_tokens"] == 4096 - - def test_echoing_the_read_back_blob_preserves_every_stored_key(self): - """The Admin UI edit form submits the whole /model/info row back, and that read reports - every key the deployment never stored as an explicit null. Those nulls have to stay - no-ops: a write drops None before storing, so a null in the echoed blob always names a - key the stored row does not carry. - """ - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - db_model = _build_db_model_with_pinned_model_info() - echoed = { - "id": "dep-pinned-0", - "max_input_tokens": 4096, - "mode": "chat", - "supports_vision": True, - "input_cost_per_token": 0.000001, - "team_id": "team-keep-me", - "base_model": None, - "tier": None, - "max_output_tokens": None, - "supports_function_calling": None, - "cache_read_input_token_cost": None, - } - - result = update_db_model( - db_model=db_model, - updated_patch=updateDeployment.model_validate({"model_info": echoed}), - ) - - info = json.loads(result["model_info"]) - assert info["max_input_tokens"] == 4096 - assert info["mode"] == "chat" - assert info["supports_vision"] is True - assert info["input_cost_per_token"] == 0.000001 - assert info["team_id"] == "team-keep-me" - for never_stored in ("base_model", "tier", "max_output_tokens", "supports_function_calling"): - assert never_stored not in info - - def test_null_on_pricing_key_still_clears_both_blobs(self): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate( - {"model_info": {"input_cost_per_token": None}} - ), - ) - - params = json.loads(result["litellm_params"]) - info = json.loads(result["model_info"]) - assert "input_cost_per_token" not in params - assert "input_cost_per_token" not in info - assert params["max_input_tokens"] == 4096 - assert info["max_input_tokens"] == 4096 - - class TestGetModelInfoWithIdBlocked: """`ProxyConfig.get_model_info_with_id` must propagate the DB-level `blocked` column into the in-memory `model_info` dict so the router filter can read it.""" diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index d22ec60e61a..30b265905f3 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -220,198 +220,6 @@ def test_should_store_full_pricing_under_deployment_model_id(): assert entry["output_cost_per_token"] == 0.0 -def test_should_drop_a_price_the_deployment_no_longer_carries(): - """Re-registering a deployment must replace its model_id entry, not merge onto it. - - A merge left the old rate in the cost map after an operator cleared the override, so - the deployment kept billing at a price its config no longer had. - """ - backend_model = "vertex_ai/gemini-2.5-flash" - model_id = "deployment-cleared-price" - original = {model_id: litellm.model_cost.get(model_id)} - - try: - Router._register_deployment_in_model_cost( - model_id=model_id, - model_info={"input_cost_per_token": 0.005, "output_cost_per_token": 0.01}, - model=backend_model, - custom_llm_provider="vertex_ai", - ) - assert litellm.model_cost[model_id]["input_cost_per_token"] == 0.005 - - Router._register_deployment_in_model_cost( - model_id=model_id, - model_info={"mode": "chat"}, - model=backend_model, - custom_llm_provider="vertex_ai", - ) - - entry = litellm.model_cost[model_id] - assert entry.get("input_cost_per_token") != 0.005, ( - "the cleared override survived re-registration, so the deployment still bills at it" - ) - assert entry.get("output_cost_per_token") != 0.01 - finally: - _restore_model_cost_entries(original) - - -def test_should_not_strip_a_builtin_entry_when_a_deployment_id_collides_with_it(): - """Deployments are keyed into the same cost map as the built-in catalog, so a deployment - whose id happens to name a real model must not evict that model's entry. - - Stripping it would take the pricing and capability flags every other deployment of that - model reads, process-wide, until the next price-map reload. Registering twice, because - the first registration is what would mark the entry as this deployment's own. - """ - colliding_id = "gpt-4o" - original = {colliding_id: litellm.model_cost.get(colliding_id)} - builtin_max_tokens = litellm.model_cost[colliding_id]["max_tokens"] - - try: - for _ in range(2): - Router._register_deployment_in_model_cost( - model_id=colliding_id, - model_info={"id": colliding_id, "db_model": True, "mode": "chat"}, - model="gpt-4o-mini", - custom_llm_provider="openai", - ) - - entry = litellm.model_cost[colliding_id] - assert entry["max_tokens"] == builtin_max_tokens, ( - "registering a deployment under a catalog model's name wiped that model's context window" - ) - assert entry["litellm_provider"] == "openai" - assert entry["supports_vision"] is True - finally: - _restore_model_cost_entries(original) - - -def test_should_drop_a_stale_price_even_when_the_deployment_declares_a_provider(): - """A deployment may carry `litellm_provider` in its own model_info, which must not be - read as "this is a catalog entry" and stop the stale price from being dropped.""" - model_id = "deployment-provider-tagged" - original = {model_id: litellm.model_cost.get(model_id)} - - try: - Router._register_deployment_in_model_cost( - model_id=model_id, - model_info={"id": model_id, "litellm_provider": "openai", "input_cost_per_token": 0.005}, - model="gpt-4o-mini", - custom_llm_provider="openai", - ) - assert litellm.model_cost[model_id]["input_cost_per_token"] == 0.005 - - Router._register_deployment_in_model_cost( - model_id=model_id, - model_info={"id": model_id, "litellm_provider": "openai", "mode": "chat"}, - model="gpt-4o-mini", - custom_llm_provider="openai", - ) - - assert litellm.model_cost[model_id].get("input_cost_per_token") != 0.005, ( - "a deployment that declares its provider kept billing at the price it no longer carries" - ) - finally: - _restore_model_cost_entries(original) - - -def test_should_give_a_cost_map_key_back_when_the_deployment_is_deleted(): - """Deleting a deployment releases its claim on the shared cost-map key. - - Held forever, a later catalog refresh that starts publishing a model under that same - name would be treated as the deleted deployment's own entry and evicted. - """ - from litellm.router import _DEPLOYMENT_COST_MAP_KEYS - - model_id = "deployment-to-delete" - original = {model_id: litellm.model_cost.get(model_id)} - router = Router( - model_list=[ - { - "model_name": "to-delete", - "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, - "model_info": {"id": model_id, "input_cost_per_token": 0.005}, - } - ] - ) - - try: - assert model_id in _DEPLOYMENT_COST_MAP_KEYS - - assert router.delete_deployment(id=model_id) is not None - - assert model_id not in _DEPLOYMENT_COST_MAP_KEYS, ( - "a deleted deployment kept its claim on the shared cost-map key" - ) - finally: - _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) - _restore_model_cost_entries(original) - - -def test_should_keep_the_cost_map_key_while_another_router_still_serves_it(): - """Two live routers can serve the same deployment id, and the claim is process-wide. - - Releasing it when only one of them drops the deployment would put the survivor back on - merging, so the price it just cleared would keep billing. - """ - from litellm.router import _DEPLOYMENT_COST_MAP_KEYS - - model_id = "deployment-served-twice" - original = {model_id: litellm.model_cost.get(model_id)} - entry = { - "model_name": "served-twice", - "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, - "model_info": {"id": model_id, "input_cost_per_token": 0.005}, - } - first = Router(model_list=[entry]) - second = Router(model_list=[entry]) - - try: - assert model_id in _DEPLOYMENT_COST_MAP_KEYS - - assert first.delete_deployment(id=model_id) is not None - - assert model_id in _DEPLOYMENT_COST_MAP_KEYS, ( - "the claim was released while another router still served the deployment" - ) - - assert second.delete_deployment(id=model_id) is not None - assert model_id not in _DEPLOYMENT_COST_MAP_KEYS - finally: - _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) - _restore_model_cost_entries(original) - - -def test_should_keep_the_cost_map_key_while_a_dynamically_built_router_serves_it(): - """A router built with no model_list still serves whatever add_deployment gives it, so it - counts when deciding whether the shared cost-map claim can be released.""" - from litellm.router import _DEPLOYMENT_COST_MAP_KEYS - - model_id = "deployment-added-dynamically" - original = {model_id: litellm.model_cost.get(model_id)} - entry = { - "model_name": "added-dynamically", - "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, - "model_info": {"id": model_id, "input_cost_per_token": 0.005}, - } - configured = Router(model_list=[entry]) - dynamic = Router() - dynamic.add_deployment(deployment=Deployment(**entry)) - - try: - assert configured.delete_deployment(id=model_id) is not None - - assert model_id in _DEPLOYMENT_COST_MAP_KEYS, ( - "the claim was released while a dynamically built router still served the deployment" - ) - - assert dynamic.delete_deployment(id=model_id) is not None - assert model_id not in _DEPLOYMENT_COST_MAP_KEYS - finally: - _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) - _restore_model_cost_entries(original) - - def test_should_preserve_builtin_pricing_regardless_of_deployment_order(): """ The built-in pricing should be preserved no matter which deployment diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 93aaf3ca58c..83b0d58f2b2 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9064,9 +9064,8 @@ export interface paths { * Patch Model * @description PATCH Endpoint for partial model updates. * - * JSON Merge Patch semantics over `litellm_params` and `model_info`: a key absent from the - * body is unchanged, a key sent as null is removed from the stored row, and a key sent with a - * value is set (identity and ownership keys such as `id` and `team_id` ignore a null). + * Only updates the fields specified in the request while preserving other existing values. + * Follows proper PATCH semantics by only modifying provided fields. * * Args: * model_id: The ID of the model to update From 9f21ae395aa298ab61331fb74e85d4769aae05e5 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 13:16:04 +0000 Subject: [PATCH 183/411] fix(registry): correct eu Claude 3.5 Haiku Bedrock pricing, add Nova v1 tool_choice, Azure gpt-5.5 snapshot retirement Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 65 ++++++++++++------- model_prices_and_context_window.json | 65 ++++++++++++------- .../test_ai_policy_suggester.py | 2 +- tests/test_litellm/test_utils.py | 40 ++++++++---- 4 files changed, 111 insertions(+), 61 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 43def22ba58..d1f1a2d2b48 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -364,7 +364,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -537,7 +538,8 @@ "output_cost_per_token": 1.4e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "amazon.nova-pro-v1:0": { "input_cost_per_token": 8e-07, @@ -551,7 +553,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "amazon.nova-sonic-v1:0": { "deprecation_date": "2026-09-14", @@ -2876,7 +2879,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "apac.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.7e-08, @@ -2888,7 +2892,8 @@ "output_cost_per_token": 1.48e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "apac.amazon.nova-pro-v1:0": { "input_cost_per_token": 8.4e-07, @@ -2902,7 +2907,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { "deprecation_date": "2026-07-30", @@ -8064,7 +8070,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8108,7 +8115,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8152,7 +8160,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -12317,7 +12326,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -12496,7 +12506,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": { "input_cost_per_token": 4.2e-08, @@ -12508,7 +12519,8 @@ "output_cost_per_token": 1.68e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, @@ -12522,7 +12534,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -20990,7 +21003,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "eu.amazon.nova-micro-v1:0": { "input_cost_per_token": 4.6e-08, @@ -21002,7 +21016,8 @@ "output_cost_per_token": 1.84e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "eu.amazon.nova-pro-v1:0": { "input_cost_per_token": 1.05e-06, @@ -21017,24 +21032,25 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { - "input_cost_per_token": 2.5e-07, + "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.25e-06, + "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07, + "cache_read_input_token_cost": 8e-08, + "cache_creation_input_token_cost": 1e-06, "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -43795,7 +43811,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "us.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.5e-08, @@ -43807,7 +43824,8 @@ "output_cost_per_token": 1.4e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "us.amazon.nova-premier-v1:0": { "deprecation_date": "2026-09-14", @@ -43836,7 +43854,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 43def22ba58..d1f1a2d2b48 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -364,7 +364,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -537,7 +538,8 @@ "output_cost_per_token": 1.4e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "amazon.nova-pro-v1:0": { "input_cost_per_token": 8e-07, @@ -551,7 +553,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "amazon.nova-sonic-v1:0": { "deprecation_date": "2026-09-14", @@ -2876,7 +2879,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "apac.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.7e-08, @@ -2888,7 +2892,8 @@ "output_cost_per_token": 1.48e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "apac.amazon.nova-pro-v1:0": { "input_cost_per_token": 8.4e-07, @@ -2902,7 +2907,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { "deprecation_date": "2026-07-30", @@ -8064,7 +8070,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8108,7 +8115,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, @@ -8152,7 +8160,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2027-10-26" }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -12317,7 +12326,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -12496,7 +12506,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": { "input_cost_per_token": 4.2e-08, @@ -12508,7 +12519,8 @@ "output_cost_per_token": 1.68e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, @@ -12522,7 +12534,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "bedrock/us-gov-west-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -20990,7 +21003,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "eu.amazon.nova-micro-v1:0": { "input_cost_per_token": 4.6e-08, @@ -21002,7 +21016,8 @@ "output_cost_per_token": 1.84e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "eu.amazon.nova-pro-v1:0": { "input_cost_per_token": 1.05e-06, @@ -21017,24 +21032,25 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { - "input_cost_per_token": 2.5e-07, + "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.25e-06, + "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07, + "cache_read_input_token_cost": 8e-08, + "cache_creation_input_token_cost": 1e-06, "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -43795,7 +43811,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "us.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.5e-08, @@ -43807,7 +43824,8 @@ "output_cost_per_token": 1.4e-07, "supports_function_calling": true, "supports_prompt_caching": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_tool_choice": true }, "us.amazon.nova-premier-v1:0": { "deprecation_date": "2026-09-14", @@ -43836,7 +43854,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_tool_choice": true }, "us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, diff --git a/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py b/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py index 93dc429168f..bb71d67f24e 100644 --- a/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py +++ b/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py @@ -265,7 +265,7 @@ class TestSuggesterRejectsModelsWithoutToolCalling: def test_a_model_without_forced_tool_choice_support_remains_eligible(self, local_model_cost_map): supported_params = litellm.get_supported_openai_params( - model="amazon.nova-pro-v1:0", + model="meta.llama4-scout-17b-instruct-v1:0", custom_llm_provider="bedrock", ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d10e7634720..ec3a141a490 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1409,23 +1409,35 @@ def test_supports_tool_choice_simple_tests(): is True ) - assert ( - litellm.utils.supports_tool_choice(model="us.amazon.nova-micro-v1:0") is False - ) - assert ( - litellm.utils.supports_tool_choice(model="bedrock/us.amazon.nova-micro-v1:0") - is False - ) - assert ( - litellm.utils.supports_tool_choice( - model="us.amazon.nova-micro-v1:0", custom_llm_provider="bedrock_converse" - ) - is False - ) - assert litellm.utils.supports_tool_choice(model="perplexity/sonar") is False +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + "model", + [ + "amazon.nova-lite-v1:0", + "amazon.nova-micro-v1:0", + "amazon.nova-pro-v1:0", + "apac.amazon.nova-lite-v1:0", + "apac.amazon.nova-micro-v1:0", + "apac.amazon.nova-pro-v1:0", + "bedrock/us-gov-east-1/amazon.nova-pro-v1:0", + "bedrock/us-gov-west-1/amazon.nova-lite-v1:0", + "bedrock/us-gov-west-1/amazon.nova-micro-v1:0", + "bedrock/us-gov-west-1/amazon.nova-pro-v1:0", + "eu.amazon.nova-lite-v1:0", + "eu.amazon.nova-micro-v1:0", + "eu.amazon.nova-pro-v1:0", + "us.amazon.nova-lite-v1:0", + "us.amazon.nova-micro-v1:0", + "us.amazon.nova-pro-v1:0", + ], +) +def test_amazon_nova_v1_understanding_models_support_tool_choice(model: str) -> None: + assert litellm.utils.supports_tool_choice(model=model) is True + + def test_check_provider_match(): """ Test the _check_provider_match function for various provider scenarios From b067729082783e02c13a7b757360caa09d1297be Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 13:51:40 +0000 Subject: [PATCH 184/411] fix(registry): add xAI Imagine Video 720p per second rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 ++++ model_prices_and_context_window.json | 4 ++++ model_prices_and_context_window.schema.json | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d1f1a2d2b48..dc720e2629a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -59849,6 +59849,7 @@ "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, "source": "https://docs.x.ai/docs/models/grok-imagine-video", "supported_modalities": [ "text", @@ -59866,6 +59867,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", @@ -59883,6 +59885,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", @@ -59900,6 +59903,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d1f1a2d2b48..dc720e2629a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -59849,6 +59849,7 @@ "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, "source": "https://docs.x.ai/docs/models/grok-imagine-video", "supported_modalities": [ "text", @@ -59866,6 +59867,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", @@ -59883,6 +59885,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", @@ -59900,6 +59903,7 @@ "output_cost_per_second": 0.08, "output_cost_per_second_1080p": 0.25, "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, "source": "https://docs.x.ai/docs/models/grok-imagine-video-1.5", "supported_modalities": [ "text", diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 47a1934a703..7ed1e7e568b 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -478,6 +478,10 @@ "type": "number", "minimum": 0 }, + "output_cost_per_second_720p": { + "type": "number", + "minimum": 0 + }, "output_cost_per_token": { "type": "number", "minimum": 0, From 2881b8cd4553f1dab28e7220e7f5a1f30e960aed Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 14:09:25 +0000 Subject: [PATCH 185/411] fix(cost): carry output_cost_per_second_720p through model info Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/router.py | 1 + litellm/types/utils.py | 2 ++ litellm/utils.py | 1 + tests/test_litellm/test_utils.py | 2 ++ tests/test_litellm/test_video_generation.py | 26 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +++ 6 files changed, 36 insertions(+) diff --git a/litellm/types/router.py b/litellm/types/router.py index 5c9eab30f3d..6b707a544a2 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -525,6 +525,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): input_cost_per_second: float | None output_cost_per_second: float | None output_cost_per_second_480p: ReadOnly[float | None] + output_cost_per_second_720p: ReadOnly[float | None] output_cost_per_second_1080p: float | None output_cost_per_second_4k: ReadOnly[float | None] num_retries: int | None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d62f00f3676..ab0cc5f959c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -318,6 +318,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): float | None ) # video_generation tier: key output_cost_per_second_ (e.g. 1080p, 720p) output_cost_per_second_480p: ReadOnly[float | None] + output_cost_per_second_720p: ReadOnly[float | None] output_cost_per_second_4k: ReadOnly[float | None] ocr_cost_per_page: float | None # for OCR models ocr_cost_per_credit: float | None # for OCR models priced by credit @@ -3522,6 +3523,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_second: float | None = None output_cost_per_second_1080p: float | None = None output_cost_per_second_480p: float | None = None + output_cost_per_second_720p: float | None = None output_cost_per_second_4k: float | None = None input_cost_per_pixel: float | None = None output_cost_per_pixel: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 36b48d3b8d8..d0ff4917616 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5913,6 +5913,7 @@ def _get_model_info_helper( output_cost_per_second=_model_info.get("output_cost_per_second", None), output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None), output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None), + output_cost_per_second_720p=_model_info.get("output_cost_per_second_720p", None), output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None), output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None), output_cost_per_image=_model_info.get("output_cost_per_image", None), diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index ec3a141a490..357c0d1c363 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -814,6 +814,7 @@ def validate_model_cost_values(model_data, exceptions=None): "input_cost_per_second", "output_cost_per_second", "output_cost_per_second_480p", + "output_cost_per_second_720p", "output_cost_per_second_1080p", "output_cost_per_second_4k", "input_cost_per_query", @@ -1037,6 +1038,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "output_cost_per_pixel": {"type": "number"}, "output_cost_per_second": {"type": "number"}, "output_cost_per_second_480p": {"type": "number"}, + "output_cost_per_second_720p": {"type": "number"}, "output_cost_per_second_1080p": {"type": "number"}, "output_cost_per_second_4k": {"type": "number"}, "output_cost_per_token": {"type": "number"}, diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 2a60ff9c4b5..f3cd4618078 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -532,6 +532,32 @@ class TestVideoGeneration: assert abs(cost_for("runwayml/seedance2_5", "480p", 8.0) - 1.6) < 0.001 assert abs(cost_for("runwayml/gen4.5", None, 8.0) - 0.96) < 0.001 + def test_completion_cost_xai_imagine_video_720p_tier_from_cost_map(self, monkeypatch): + """720p xAI Imagine Video requests bill the published 720p rate, not the 480p base rate.""" + from litellm.cost_calculator import completion_cost + + local_map_path = os.path.join( + os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json" + ) + with open(local_map_path, "r") as f: + monkeypatch.setattr(litellm, "model_cost", json.load(f)) + + def cost_for(model: str, resolution: str, duration: float) -> float: + mock_response = MagicMock() + mock_response.usage = {"duration_seconds": duration, "video_resolution": resolution} + type(mock_response)._hidden_params = {} + return completion_cost( + completion_response=mock_response, + model=model, + call_type="create_video", + custom_llm_provider="xai", + ) + + assert abs(cost_for("xai/grok-imagine-video", "720p", 10.0) - 0.7) < 0.001 + assert abs(cost_for("xai/grok-imagine-video-1.5", "720p", 10.0) - 1.4) < 0.001 + assert abs(cost_for("xai/grok-imagine-video-1.5", "480p", 10.0) - 0.8) < 0.001 + assert abs(cost_for("xai/grok-imagine-video-1.5", "1080p", 10.0) - 2.5) < 0.001 + def test_completion_cost_veo_31_tiers_pin_published_rates(self, monkeypatch): """The gemini and vertex_ai veo 3.1 entries bill Google's published per-second tier rates.""" from litellm.cost_calculator import completion_cost diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 83b0d58f2b2..ce50978602f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29629,6 +29629,8 @@ export interface components { output_cost_per_second_480p?: number | null; /** Output Cost Per Second 4K */ output_cost_per_second_4k?: number | null; + /** Output Cost Per Second 720P */ + output_cost_per_second_720p?: number | null; /** Output Cost Per Token */ output_cost_per_token?: number | null; /** Output Cost Per Token Above 128K Tokens */ @@ -39820,6 +39822,8 @@ export interface components { output_cost_per_second_480p?: number | null; /** Output Cost Per Second 4K */ output_cost_per_second_4k?: number | null; + /** Output Cost Per Second 720P */ + output_cost_per_second_720p?: number | null; /** Output Cost Per Token */ output_cost_per_token?: number | null; /** Output Cost Per Token Above 128K Tokens */ From a1588c260232dd6ed22605fdd4db24983bb1fcee Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:05:15 -0700 Subject: [PATCH 186/411] fix(mcp): preserve stream failures across transports --- litellm/experimental_mcp_client/client.py | 7 +- .../mcp_server/rest_endpoints.py | 25 +- .../test_mcp_client.py | 240 ++++++++++++++++++ .../mcp_server/test_rest_endpoints.py | 48 ++++ 4 files changed, 313 insertions(+), 7 deletions(-) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index cabcc6c03ba..5cdc4efdfef 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -451,7 +451,7 @@ class MCPClient: async def receive_message( message: RequestResponder[ServerRequest, ClientResult] | ServerNotification | Exception, ) -> None: - if not isinstance(message, ValueError): + if not isinstance(message, (ValueError, httpx.RequestError, OSError)): return if not stream_error.done(): stream_error.set_result(message) @@ -472,7 +472,7 @@ class MCPClient: read_stream, write_stream, read_timeout_seconds=timedelta(seconds=self.timeout), - message_handler=receive_message if self.transport_type == MCPTransport.http else None, + message_handler=receive_message, **session_kwargs, ) session: Final = await session_ctx.__aenter__() @@ -525,8 +525,7 @@ class MCPClient: read_timeout: Final = _as_read_timeout(e) if read_timeout is not None: verbose_logger.warning( - "MCP client timed out after %ss waiting for %s to answer; the server accepted the " - "request and ended its response stream without a JSON-RPC reply", + "MCP client timed out after %ss waiting for a valid MCP response from %s", self.timeout, self.server_url or "stdio", ) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index c0ffeaeac62..63badc6f703 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -114,7 +114,8 @@ def _known_connection_error_message(exc: BaseException, url: str | None, timeout return str(exc.detail) if isinstance(exc, TimeoutError): return ( - f"Failed to connect to MCP server: no response from {_redact_mcp_resource_url(url) or 'the server'} " + "Failed to connect to MCP server: no valid MCP response received from " + f"{_redact_mcp_resource_url(url) or 'the server'} " f"within {timeout_seconds:.0f}s. Check that the LiteLLM proxy can reach this URL " "from its network (DNS, egress rules, firewalls) and that the server answers MCP requests." ) @@ -131,6 +132,11 @@ def _known_connection_error_message(exc: BaseException, url: str | None, timeout return "Failed to connect to MCP server: the connection timed out." if isinstance(exc, httpx.HTTPStatusError): return f"Failed to connect to MCP server: it returned HTTP {exc.response.status_code}." + if isinstance(exc, (httpx.NetworkError, httpx.RemoteProtocolError, ConnectionError)): + return ( + "Failed to connect to MCP server: the connection was interrupted. " + "Check the server and network connection, then retry." + ) if isinstance(exc, ValueError) and str(exc).startswith("Unexpected content type:"): return ( "Failed to connect to MCP server: the endpoint returned an unsupported content type. " @@ -142,6 +148,11 @@ def _known_connection_error_message(exc: BaseException, url: str | None, timeout "Check the MCP endpoint URL and the server's protocol implementation." ) if MCP_AVAILABLE and isinstance(exc, McpError): + if exc.error.code == -32000 and exc.error.message == "Connection closed": + return ( + "Failed to connect to MCP server: the connection was closed before the request completed. " + "Check that the server stays running and returns a complete MCP response, then retry." + ) if exc.error.code == 32600 and exc.error.message == "Session terminated": return ( "Failed to connect to MCP server: the MCP session was terminated. " @@ -159,7 +170,7 @@ if MCP_AVAILABLE: from mcp.shared.exceptions import McpError from mcp.types import Tool as MCPTool - from litellm.experimental_mcp_client.client import MCPClient + from litellm.experimental_mcp_client.client import MCPClient, _as_read_timeout from litellm.llms.litellm_proxy.skills.skill_search import ( DEFAULT_SKILL_SEARCH_TOP_K, ) @@ -1396,10 +1407,18 @@ if MCP_AVAILABLE: except (KeyboardInterrupt, SystemExit, asyncio.CancelledError): raise except BaseException as e: + effective_timeout: Final = ( + min(request.timeout or MCP_CLIENT_TIMEOUT, timeout_seconds) + if any( + isinstance(cause, McpError) and _as_read_timeout(cause) is not None + for cause in iter_exception_tree(e) + ) + else timeout_seconds + ) return { "status": "error", "error": True, - "message": _connection_error_message(e, request.url, timeout_seconds), + "message": _connection_error_message(e, request.url, effective_timeout), } async def _preview_openapi_tools(spec_path: str) -> dict: diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index 90292b9b162..a1e781d17b7 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -3,6 +3,7 @@ import base64 import json import os import sys +from collections.abc import AsyncIterator from importlib import metadata from pathlib import Path from typing import Final @@ -18,6 +19,7 @@ from pydantic import ValidationError from mcp.shared.message import SessionMessage from mcp.types import ( LATEST_PROTOCOL_VERSION, + CallToolResult, ErrorData, Implementation, InitializeResult, @@ -36,6 +38,7 @@ from litellm.experimental_mcp_client.client import ( MCPClient, _as_read_timeout, _first_non_cancelled_cause, + _TransportContext, missing_streamable_http_client_error, strip_auth_scheme, ) @@ -1296,6 +1299,7 @@ def test_a_differently_cased_injected_header_cannot_shadow_the_slot() -> None: [ ("text/html", b"secret-page", ValueError), ("application/json", b"secret-invalid-json", ValidationError), + ("application/json", b"", ValidationError), ("application/json", b'{"secret":"invalid-rpc"}', ValidationError), ("application/json", b'{"jsonrpc":"2.0","id":0,"result":{"secret":"invalid-schema"}}', ValidationError), ], @@ -1446,3 +1450,239 @@ async def test_invalid_tool_list_schema_is_identified_as_an_upstream_response() message: Final = _connection_error_message(caught.value, client.server_url, 30) assert "invalid MCP response" in message assert "secret" not in message + + +class _DiagnosticSSEStream(httpx.AsyncByteStream): + def __init__(self, messages: asyncio.Queue[bytes | Exception | None]) -> None: + self.messages = messages + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b"event: endpoint\ndata: /messages\n\n" + while True: + message: Final = await self.messages.get() + if message is None: + return + if isinstance(message, Exception): + raise message + yield b"event: message\ndata: " + message + b"\n\n" + + +_DIAGNOSTIC_STDIO_SERVER: Final = """ +import json, sys +mode, failure_method = sys.argv[1:] +for line in sys.stdin: + request = json.loads(line) + if "method" not in request or "id" not in request: + continue + if request["method"] == failure_method: + if mode == "bad-json": + print("secret-invalid-json", flush=True) + continue + if mode == "closed": + sys.exit(0) + if mode == "silent": + print(json.dumps({"jsonrpc": "2.0", "method": "notifications/message", "params": {"level": "info", "data": "Waiting"}}), flush=True) + continue + if request["method"] == "initialize": + result = {"protocolVersion": request["params"]["protocolVersion"], "capabilities": {"tools": {}, "logging": {}}, "serverInfo": {"name": "diagnostic", "version": "1"}} + elif request["method"] == "tools/list": + print(json.dumps({"jsonrpc": "2.0", "method": "notifications/message", "params": {"level": "info", "data": "Listing tools"}}), flush=True) + print(json.dumps({"jsonrpc": "2.0", "id": "unmatched", "result": {}}), flush=True) + print(json.dumps({"jsonrpc": "2.0", "id": "server-ping", "method": "ping"}), flush=True) + result = {"tools": [{"name": "ping", "inputSchema": {"type": "object"}}]} + else: + result = {"content": [{"type": "text", "text": "pong"}], "isError": False} + print(json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": result}), flush=True) +""" + + +def _diagnostic_transport(transport: MCPTransport, mode: str, failure_method: str) -> _TransportContext: + from mcp import StdioServerParameters + from mcp.client.sse import sse_client + from mcp.client.stdio import stdio_client + + if transport == MCPTransport.stdio: + return stdio_client( + StdioServerParameters( + command=sys.executable, args=["-u", "-c", _DIAGNOSTIC_STDIO_SERVER, mode, failure_method] + ) + ) + messages: Final[asyncio.Queue[bytes | Exception | None]] = asyncio.Queue() + + async def respond(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + return httpx.Response( + 200, headers={"Content-Type": "text/event-stream"}, stream=_DiagnosticSSEStream(messages) + ) + payload: Final = json.loads(request.content) + if "method" not in payload or "id" not in payload: + return httpx.Response(202) + if payload["method"] == failure_method and mode != "ok": + if mode == "bad-json": + await messages.put(b"secret-invalid-json") + elif mode == "io-error": + await messages.put(httpx.ReadError("secret-read-error")) + elif mode == "closed": + await messages.put(None) + elif mode == "silent": + await messages.put( + b'{"jsonrpc":"2.0","method":"notifications/message","params":{"level":"info","data":"Waiting"}}' + ) + return httpx.Response(202) + if payload["method"] == "tools/list": + for message in ( + { + "jsonrpc": "2.0", + "method": "notifications/message", + "params": {"level": "info", "data": "Listing tools"}, + }, + {"jsonrpc": "2.0", "id": "unmatched", "result": {}}, + {"jsonrpc": "2.0", "id": "server-ping", "method": "ping"}, + ): + await messages.put(json.dumps(message).encode()) + result: Final = ( + { + "protocolVersion": LATEST_PROTOCOL_VERSION, + "capabilities": {"tools": {}, "logging": {}}, + "serverInfo": {"name": "diagnostic", "version": "1"}, + } + if payload["method"] == "initialize" + else {"tools": [{"name": "ping", "inputSchema": {"type": "object"}}]} + if payload["method"] == "tools/list" + else {"content": [{"type": "text", "text": "pong"}], "isError": False} + ) + await messages.put(json.dumps({"jsonrpc": "2.0", "id": payload["id"], "result": result}).encode()) + return httpx.Response(202) + + def factory( + headers: dict[str, str] | None = None, + timeout: httpx.Timeout | None = None, + auth: httpx.Auth | None = None, + ) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.MockTransport(respond), headers=headers, timeout=timeout, auth=auth) + + return sse_client("https://example.com/sse", httpx_client_factory=factory) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) +@pytest.mark.parametrize("failure_method", ["initialize", "tools/list"]) +async def test_transport_parsing_failure_is_preserved(transport: MCPTransport, failure_method: str) -> None: + client: Final = MCPClient(server_url="https://example.com/sse", transport_type=transport, timeout=0.2) + with pytest.raises(ValidationError): + await asyncio.wait_for( + client._execute_session_operation( + _diagnostic_transport(transport, "bad-json", failure_method), lambda session: session.list_tools() + ), + timeout=3, + ) + + +@pytest.mark.asyncio +async def test_sse_read_failure_is_preserved() -> None: + client: Final = MCPClient(server_url="https://example.com/sse", transport_type=MCPTransport.sse, timeout=0.2) + with pytest.raises(httpx.ReadError, match="secret-read-error"): + await asyncio.wait_for( + client._execute_session_operation( + _diagnostic_transport(MCPTransport.sse, "io-error", "tools/list"), lambda session: session.list_tools() + ), + timeout=3, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) +@pytest.mark.parametrize("mode", ["ok", "closed", "silent"]) +async def test_transport_completion_and_normal_messages(transport: MCPTransport, mode: str) -> None: + from mcp import ClientSession + from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message + + logging_callback: Final = AsyncMock() + client: Final = MCPClient( + server_url="https://example.com/sse", transport_type=transport, timeout=0.2, logging_callback=logging_callback + ) + + async def operation(session: ClientSession) -> CallToolResult: + tools: Final = await session.list_tools() + assert [tool.name for tool in tools.tools] == ["ping"] + return await session.call_tool("ping", {}) + + pending: Final = client._execute_session_operation(_diagnostic_transport(transport, mode, "tools/list"), operation) + if mode == "ok": + result: Final = await asyncio.wait_for(pending, timeout=3) + assert result.isError is False + assert result.content[0].text == "pong" + logging_callback.assert_awaited_once_with(LoggingMessageNotificationParams(level="info", data="Listing tools")) + else: + with pytest.raises(McpError) as caught: + await asyncio.wait_for(pending, timeout=3) + if mode == "closed": + assert "connection was closed" in _connection_error_message(caught.value, client.server_url, 0.2) + else: + assert isinstance(_as_read_timeout(caught.value), TimeoutError) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) +async def test_transport_cancellation_cleans_up_a_pending_request(transport: MCPTransport) -> None: + ready: Final = asyncio.Event() + + async def on_log(message: LoggingMessageNotificationParams) -> None: + if message.data == "Waiting": + ready.set() + + client: Final = MCPClient( + server_url="https://example.com/sse", transport_type=transport, timeout=30, logging_callback=on_log + ) + task: Final = asyncio.create_task( + client._execute_session_operation( + _diagnostic_transport(transport, "silent", "tools/list"), lambda session: session.list_tools() + ) + ) + try: + await asyncio.wait_for(ready.wait(), timeout=3) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=3) + + +class _InterruptedHTTPBody(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'{"jsonrpc":' + raise httpx.RemoteProtocolError("secret-incomplete-response") + + +@pytest.mark.asyncio +async def test_interrupted_http_response_preserves_the_transport_failure() -> None: + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"Content-Type": "application/json"}, stream=_InterruptedHTTPBody()) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) + with pytest.raises(httpx.RemoteProtocolError, match="secret-incomplete-response"): + await asyncio.wait_for( + client._execute_session_operation( + streamable_http_client(client.server_url, http_client=http_client), + lambda session: session.list_tools(), + ), + timeout=3, + ) + + +@pytest.mark.asyncio +async def test_empty_http_event_stream_uses_the_existing_request_deadline() -> None: + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=b"") + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + client: Final = MCPClient(server_url="https://example.com/mcp", timeout=0.2) + with pytest.raises(McpError) as caught: + await asyncio.wait_for( + client._execute_session_operation( + streamable_http_client(client.server_url, http_client=http_client), + lambda session: session.list_tools(), + ), + timeout=3, + ) + assert isinstance(_as_read_timeout(caught.value), TimeoutError) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 73734131130..101b02e6ffd 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -3427,6 +3427,54 @@ class TestConnectionErrorMessage: message = rest_endpoints._connection_error_message(exc, "https://example.com", 30.0) assert "503" in message + @pytest.mark.parametrize( + "error_type", [httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError, ConnectionResetError] + ) + def test_interrupted_connection_message_is_safe(self, error_type: type[Exception]) -> None: + message: Final = rest_endpoints._connection_error_message( + error_type("secret-transport-detail"), "https://example.com/?token=secret-query", 30 + ) + assert "connection was interrupted" in message + assert "secret" not in message + + def test_closed_connection_explains_incomplete_request(self) -> None: + from mcp import McpError + from mcp.types import ErrorData + + message: Final = rest_endpoints._connection_error_message( + McpError(ErrorData(code=-32000, message="Connection closed", data="secret-data")), None, 30 + ) + assert "connection was closed before the request completed" in message + assert "secret" not in message + + def test_timeout_does_not_claim_the_server_sent_nothing(self) -> None: + message: Final = rest_endpoints._connection_error_message(TimeoutError(), None, 30) + assert "no valid MCP response received" in message + + @pytest.mark.asyncio + @pytest.mark.parametrize("sdk_timeout", [True, False]) + async def test_timeout_message_uses_the_deadline_that_expired(self, sdk_timeout: bool) -> None: + from mcp import McpError + from mcp.types import ErrorData + + async def operation(client: rest_endpoints.MCPClient) -> dict[str, object]: + try: + raise TimeoutError("secret-timeout") + except TimeoutError as elapsed: + if not sdk_timeout: + raise + try: + raise McpError(ErrorData(code=408, message="secret-sdk-timeout")) from elapsed + except McpError as sdk_error: + raise TimeoutError() from sdk_error + + payload: Final = NewMCPServerRequest( + server_name="timeout", url="https://example.com", auth_type=MCPAuth.none, timeout=1 + ) + result: Final = await rest_endpoints._execute_with_mcp_client(payload, operation, timeout_seconds=30) + assert ("within 1s" if sdk_timeout else "within 30s") in result["message"] + assert "secret" not in result["message"] + def test_unknown_error_falls_back_to_generic(self): message = rest_endpoints._connection_error_message(RuntimeError("weird"), "https://example.com", 30.0) assert "weird" not in message From 1fc1aaabdaa3542cd2d410103ee01ebc94025961 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 16:00:35 +0000 Subject: [PATCH 187/411] test(cli): drop lite --version subprocess regression Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/client/cli/test_global_options.py | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 9ed9f91ab9f..b73d1acc6e3 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -1,10 +1,6 @@ # stdlib imports import json import os -import shutil -import subprocess -import threading -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from unittest.mock import Mock, patch @@ -37,58 +33,6 @@ def test_cli_version_flag(cli_runner): assert "LiteLLM Proxy Server Version: 1.2.3" in result.output -def test_lite_version_does_not_fetch_model_cost_map(tmp_path: Path) -> None: - lite_path = shutil.which("lite") - if lite_path is None: - pytest.skip("lite executable is unavailable") - - request_log: Path = tmp_path / "requests.log" - - class _CostMapHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - with request_log.open("a", encoding="utf-8") as log_file: - log_file.write(f"{self.path}\n") - body = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: object) -> None: - return - - server = ThreadingHTTPServer(("127.0.0.1", 0), _CostMapHandler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - env = {key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"} - source_root = str(Path(__file__).resolve().parents[5]) - env["PYTHONPATH"] = os.pathsep.join(filter(None, (source_root, env.get("PYTHONPATH")))) - env.update( - { - "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", - "LITELLM_PROXY_URL": "http://127.0.0.1:9", - } - ) - result = subprocess.run( - [lite_path, "--version"], - capture_output=True, - text=True, - timeout=120, - env=env, - ) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=10) - - assert result.returncode == 0 - assert "LiteLLM Proxy CLI Version" in result.stdout - request_count: int = request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 - assert request_count == 0 - - def test_cli_source_is_ascii_only(): """Non-ASCII output (emoji, box-drawing chars) raises UnicodeEncodeError on legacy Windows consoles (cp1252), so the whole CLI package must stay ASCII-only.""" From 9b6c7c8bf0e2e0e3218caa8a18b29b17bb8bf566 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:17:14 -0700 Subject: [PATCH 188/411] Apply suggestion from @greptile-apps[bot] Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/proxy/_experimental/mcp_server/rest_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 63badc6f703..31252479445 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1408,7 +1408,7 @@ if MCP_AVAILABLE: raise except BaseException as e: effective_timeout: Final = ( - min(request.timeout or MCP_CLIENT_TIMEOUT, timeout_seconds) + min(request.timeout if request.timeout is not None else MCP_CLIENT_TIMEOUT, timeout_seconds) if any( isinstance(cause, McpError) and _as_read_timeout(cause) is not None for cause in iter_exception_tree(e) From 6dbaf43ba2951e403fd1efe6920eaf6c30550c42 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 16:42:07 +0000 Subject: [PATCH 189/411] fix(registry): drop OpenAI shutdown date from shared computer-use-preview entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 1 - model_prices_and_context_window.json | 1 - 2 files changed, 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dc720e2629a..cdd9fc7b6d4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14560,7 +14560,6 @@ "supports_tool_choice": true }, "computer-use-preview": { - "deprecation_date": "2026-07-23", "input_cost_per_token": 3e-06, "litellm_provider": "azure", "max_input_tokens": 8192, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dc720e2629a..cdd9fc7b6d4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14560,7 +14560,6 @@ "supports_tool_choice": true }, "computer-use-preview": { - "deprecation_date": "2026-07-23", "input_cost_per_token": 3e-06, "litellm_provider": "azure", "max_input_tokens": 8192, From dbf94902294cf5fa4fe5af6681fcd83ffd743231 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:44:14 -0700 Subject: [PATCH 190/411] fix(mcp): expose shared SDK timeout normalization --- litellm/experimental_mcp_client/client.py | 6 +++--- .../_experimental/mcp_server/rest_endpoints.py | 4 ++-- .../experimental_mcp_client/test_mcp_client.py | 18 +++++++++--------- .../mcp_server/test_rest_endpoints.py | 7 ++++--- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 5cdc4efdfef..8c7f4557992 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -150,8 +150,8 @@ _SDK_READ_TIMEOUT_CODE: Final = int(httpx.codes.REQUEST_TIMEOUT) otherwise carries JSON-RPC error codes.""" -def _as_read_timeout(exc: BaseException) -> TimeoutError | None: - """The session read timeout elapsing, re-expressed as a ``TimeoutError``, or ``None``. +def as_mcp_read_timeout(exc: BaseException) -> TimeoutError | None: + """Normalize an MCP SDK read timeout for client and gateway diagnostics, or return ``None``. The SDK reports its own elapsed read timeout as ``McpError`` carrying an HTTP status code in a field that otherwise holds JSON-RPC error codes, and it relays an upstream's JSON-RPC error @@ -522,7 +522,7 @@ class MCPClient: transport_ctx, http_client = self._create_transport_context() return await self._execute_session_operation(transport_ctx, operation) except Exception as e: - read_timeout: Final = _as_read_timeout(e) + read_timeout: Final = as_mcp_read_timeout(e) if read_timeout is not None: verbose_logger.warning( "MCP client timed out after %ss waiting for a valid MCP response from %s", diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 31252479445..5b74caee3a2 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -170,7 +170,7 @@ if MCP_AVAILABLE: from mcp.shared.exceptions import McpError from mcp.types import Tool as MCPTool - from litellm.experimental_mcp_client.client import MCPClient, _as_read_timeout + from litellm.experimental_mcp_client.client import MCPClient, as_mcp_read_timeout from litellm.llms.litellm_proxy.skills.skill_search import ( DEFAULT_SKILL_SEARCH_TOP_K, ) @@ -1410,7 +1410,7 @@ if MCP_AVAILABLE: effective_timeout: Final = ( min(request.timeout if request.timeout is not None else MCP_CLIENT_TIMEOUT, timeout_seconds) if any( - isinstance(cause, McpError) and _as_read_timeout(cause) is not None + isinstance(cause, McpError) and as_mcp_read_timeout(cause) is not None for cause in iter_exception_tree(e) ) else timeout_seconds diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index a1e781d17b7..b07e5876e8b 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -36,9 +36,9 @@ import litellm.experimental_mcp_client.client as mcp_client_module from litellm.experimental_mcp_client.client import ( MCP_STREAMABLE_HTTP_REQUIREMENT, MCPClient, - _as_read_timeout, _first_non_cancelled_cause, _TransportContext, + as_mcp_read_timeout, missing_streamable_http_client_error, strip_auth_scheme, ) @@ -867,25 +867,25 @@ def _raise_mcp_error_while_handling_a_timeout(code: int, message: str) -> McpErr return raised -def test_as_read_timeout_separates_the_sdk_timeout_from_a_relayed_upstream_error(): +def test_as_mcp_read_timeout_separates_the_sdk_timeout_from_a_relayed_upstream_error(): """Neither signal alone is enough. The code alone cannot separate the SDK's own timeout from an upstream JSON-RPC error that happens to use 408, and the context chain alone cannot separate it from any other relayed error that surfaces while a timeout is being handled, so both must hold. """ timeout_code = int(httpx.codes.REQUEST_TIMEOUT) - translated = _as_read_timeout(_raise_mcp_error_while_handling_a_timeout(timeout_code, "Timed out while waiting")) + translated = as_mcp_read_timeout(_raise_mcp_error_while_handling_a_timeout(timeout_code, "Timed out while waiting")) assert isinstance(translated, TimeoutError) assert str(translated) == "Timed out while waiting" relayed_408 = McpError(ErrorData(code=timeout_code, message="upstream said 408")) - assert _as_read_timeout(relayed_408) is None, "an upstream 408 with no elapsed timeout is not our timeout" + assert as_mcp_read_timeout(relayed_408) is None, "an upstream 408 with no elapsed timeout is not our timeout" relayed_other = _raise_mcp_error_while_handling_a_timeout(-32603, "upstream internal error") - assert _as_read_timeout(relayed_other) is None, "a non-timeout code is not our timeout, whatever the chain" + assert as_mcp_read_timeout(relayed_other) is None, "a non-timeout code is not our timeout, whatever the chain" - assert _as_read_timeout(McpError(ErrorData(code=-32603, message="boom"))) is None - assert _as_read_timeout(RuntimeError("not an McpError")) is None + assert as_mcp_read_timeout(McpError(ErrorData(code=-32603, message="boom"))) is None + assert as_mcp_read_timeout(RuntimeError("not an McpError")) is None @pytest.mark.asyncio @@ -1619,7 +1619,7 @@ async def test_transport_completion_and_normal_messages(transport: MCPTransport, if mode == "closed": assert "connection was closed" in _connection_error_message(caught.value, client.server_url, 0.2) else: - assert isinstance(_as_read_timeout(caught.value), TimeoutError) + assert isinstance(as_mcp_read_timeout(caught.value), TimeoutError) @pytest.mark.asyncio @@ -1685,4 +1685,4 @@ async def test_empty_http_event_stream_uses_the_existing_request_deadline() -> N ), timeout=3, ) - assert isinstance(_as_read_timeout(caught.value), TimeoutError) + assert isinstance(as_mcp_read_timeout(caught.value), TimeoutError) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 101b02e6ffd..3487f634251 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -3453,7 +3453,8 @@ class TestConnectionErrorMessage: @pytest.mark.asyncio @pytest.mark.parametrize("sdk_timeout", [True, False]) - async def test_timeout_message_uses_the_deadline_that_expired(self, sdk_timeout: bool) -> None: + @pytest.mark.parametrize("read_timeout", [0, 1]) + async def test_timeout_message_uses_the_deadline_that_expired(self, sdk_timeout: bool, read_timeout: int) -> None: from mcp import McpError from mcp.types import ErrorData @@ -3469,10 +3470,10 @@ class TestConnectionErrorMessage: raise TimeoutError() from sdk_error payload: Final = NewMCPServerRequest( - server_name="timeout", url="https://example.com", auth_type=MCPAuth.none, timeout=1 + server_name="timeout", url="https://example.com", auth_type=MCPAuth.none, timeout=read_timeout ) result: Final = await rest_endpoints._execute_with_mcp_client(payload, operation, timeout_seconds=30) - assert ("within 1s" if sdk_timeout else "within 30s") in result["message"] + assert (f"within {read_timeout}s" if sdk_timeout else "within 30s") in result["message"] assert "secret" not in result["message"] def test_unknown_error_falls_back_to_generic(self): From 1183b2abc6645a92f9a852daa7843662b6fa6f20 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Wed, 9 Sep 2026 09:46:37 -0700 Subject: [PATCH 191/411] fix(integrations): pass original request object to post-call guardrail hooks (#40414) --- litellm/integrations/custom_guardrail.py | 30 ++++--- .../integrations/test_custom_guardrail.py | 90 ++++++++++++++++++- 2 files changed, 106 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 37d6a7e793d..77bf4820a1a 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -850,20 +850,24 @@ class CustomGuardrail(CustomLogger): if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True: return None - # CHECK IF GUARDRAIL REJECTS THE REQUEST target: Final = self._deployment_hook_target() - hook_request_data: Final = {**request_data, "guardrail_to_apply": self} if target is not self else request_data - result: Final = await target.async_post_call_success_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id=request_data.get("user_api_key_user_id"), - team_id=request_data.get("user_api_key_team_id"), - end_user_id=request_data.get("user_api_key_end_user_id"), - api_key=request_data.get("user_api_key_hash"), - request_route=request_data.get("user_api_key_request_route"), - ), - data=hook_request_data, - response=response, - ) + try: + if target is not self: + request_data["guardrail_to_apply"] = self # rebind-ok: dispatch consumes this key + result: Final = await target.async_post_call_success_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id=request_data.get("user_api_key_user_id"), + team_id=request_data.get("user_api_key_team_id"), + end_user_id=request_data.get("user_api_key_end_user_id"), + api_key=request_data.get("user_api_key_hash"), + request_route=request_data.get("user_api_key_request_route"), + ), + data=request_data, + response=response, + ) + finally: + if target is not self: + request_data.pop("guardrail_to_apply", None) if not self._is_valid_response_type(result): return None diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index cd8d609cf71..ddc8439a83a 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -1842,12 +1842,14 @@ class _ApplyStyleGuardrail(CustomGuardrail): self.block = block self.apply_called = False self.seen_texts = None + self.seen_request_data = None async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): from fastapi import HTTPException self.apply_called = True self.seen_texts = inputs.get("texts") + self.seen_request_data = request_data if self.block: raise HTTPException(status_code=400, detail={"error": "Violated moderation policy"}) return inputs @@ -2646,6 +2648,91 @@ class TestCustomGuardrailPostCallSuccessDeploymentHook: which starved every later callback in litellm.callbacks (notably the lazily-appended VectorStorePreCallHook that attaches provider_specific_fields["search_results"]).""" + @pytest.mark.asyncio + async def test_apply_guardrail_retains_request_identity(self) -> None: + from litellm.types.guardrails import GuardrailEventHooks + from litellm.types.utils import Choices, Message, ModelResponse + + guardrail: Final = _ApplyStyleGuardrail(block=False) + guardrail.event_hook = GuardrailEventHooks.post_call + request_data: Final = {"guardrails": ["apply-style-guardrail"]} + response: Final = ModelResponse(choices=[Choices(message=Message(content="review me"))]) + + await guardrail.async_post_call_success_deployment_hook( + request_data=request_data, response=response, call_type=CallTypes.acompletion + ) + + assert guardrail.seen_request_data is request_data + assert guardrail.seen_texts == ["review me"] + assert "guardrail_to_apply" not in request_data + + @pytest.mark.asyncio + @pytest.mark.parametrize("call_type", (None, CallTypes.acompletion)) + async def test_apply_guardrail_masks_response_and_records_metadata(self, call_type: CallTypes | None) -> None: + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, + ) + from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks + from litellm.types.utils import Choices, Message, ModelResponse + + guardrail: Final = ContentFilterGuardrail( + guardrail_name="response-filter", + event_hook=GuardrailEventHooks.post_call, + blocked_words=[BlockedWord(keyword="secret", action=ContentFilterAction.MASK)], + ) + request_data: Final = {"guardrails": ["response-filter"]} + response: Final = ModelResponse(choices=[Choices(message=Message(content="a secret"))]) + + result: Final = await guardrail.async_post_call_success_deployment_hook( + request_data=request_data, response=response, call_type=call_type + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == f"a {guardrail.keyword_redaction_tag}" + entries: Final = _guardrail_entries(request_data) + assert len(entries) == 1 + assert entries[0]["guardrail_name"] == "response-filter" + assert entries[0]["guardrail_mode"] == "post_call" + assert "guardrail_to_apply" not in request_data + + @pytest.mark.asyncio + @pytest.mark.parametrize("error_type", (None, RuntimeError, asyncio.CancelledError)) + async def test_dispatch_cleans_up_request_on_every_exit(self, error_type: type[BaseException] | None) -> None: + from contextlib import nullcontext + + from litellm.integrations.custom_logger import CustomLogger + from litellm.types.utils import LLMResponseTypes, ModelResponse + + error: Final = error_type("dispatch interrupted") if error_type is not None else None + + class Dispatch(CustomLogger): + request_data: dict[str, object] | None = None + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes + ) -> LLMResponseTypes: + self.request_data = data + if error is not None: + raise error + return response + + dispatch: Final = Dispatch() + + class Guardrail(_ApplyStyleGuardrail): + def _deployment_hook_target(self) -> CustomLogger: + return dispatch + + guardrail: Final = Guardrail(block=False) + guardrail.event_hook = GuardrailEventHooks.post_call + request_data: Final = {"guardrails": ["apply-style-guardrail"]} + with pytest.raises(error_type) if error_type is not None else nullcontext(): + await guardrail.async_post_call_success_deployment_hook( + request_data=request_data, response=ModelResponse(), call_type=CallTypes.acompletion + ) + + assert dispatch.request_data is request_data + assert "guardrail_to_apply" not in request_data + @pytest.mark.asyncio async def test_returns_none_when_request_has_no_guardrails(self): from litellm.types.utils import ModelResponse @@ -2740,4 +2827,5 @@ class TestCustomGuardrailPostCallSuccessDeploymentHook: assert result is response assert response.choices[0].message.content == "filtered response" - assert request_data == {"guardrails": ["test-guardrail"]} + assert "guardrail_to_apply" not in request_data + assert len(_guardrail_entries(request_data)) == 1 From 36f3ca95d8bdb6b03ab4632982cdc7c0cef99c6f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 9 Sep 2026 09:57:38 -0700 Subject: [PATCH 192/411] fix(proxy): report /cost/estimate rates from the call that billed them The estimate looked the reported per-token rates up a second time, with the provider this endpoint resolved rather than the one completion_cost infers. The provider decides whether a token tier threshold is inclusive, so an unrouted xai model sitting exactly on 200k billed at the tier rate and reported the base rate, half of it. completion_cost now hands back the rates its own lines were billed at, and the endpoint reports those. Claude-Session: https://claude.ai/code/session_01RLKy5DMi3XCBUJ37WzfNi1 --- litellm/cost_calculator.py | 6 ++ litellm/litellm_core_utils/litellm_logging.py | 5 ++ .../litellm_core_utils/llm_cost_calc/utils.py | 60 ++++++++++--------- .../cost_tracking_settings.py | 29 ++++----- .../llm_cost_calc/test_llm_cost_calc_utils.py | 52 +++++++++++++--- .../test_cost_tracking_settings.py | 39 ++++++++++++ tests/test_litellm/test_cost_calculator.py | 56 +++++++++++++++++ 7 files changed, 195 insertions(+), 52 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a1181ee4124..b4be023540f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import TranscriptionUsageObjectTransformation, ) from litellm.litellm_core_utils.llm_cost_calc.utils import ( + BilledTokenRates, CostCalculatorUtils, _generic_cost_per_character, _get_regional_uplift_multiplier, @@ -1122,6 +1123,7 @@ def _store_cost_breakdown_in_logging_obj( service_tier: str | None = None, data_residency: str | None = None, vertex_location: str | None = None, + billed_token_rates: BilledTokenRates | None = None, ) -> None: """ Helper function to store cost breakdown in the logging object. @@ -1166,6 +1168,7 @@ def _store_cost_breakdown_in_logging_obj( service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, + billed_token_rates=billed_token_rates, ) except Exception as breakdown_error: @@ -1735,6 +1738,7 @@ def completion_cost( _reasoning_cost: float | None = None _cache_read_cost: float | None = None _cache_creation_cost: float | None = None + _billed_token_rates: BilledTokenRates | None = None if cost_per_token_usage_object is not None and model: _breakdown_provider: str | None = ( custom_llm_provider if isinstance(custom_llm_provider, str) else None @@ -1751,6 +1755,7 @@ def completion_cost( _reasoning_cost = _token_type_breakdown.reasoning_cost _cache_read_cost = _token_type_breakdown.cache_read_cost _cache_creation_cost = _token_type_breakdown.cache_creation_cost + _billed_token_rates = _token_type_breakdown.rates _store_cost_breakdown_in_logging_obj( litellm_logging_obj=litellm_logging_obj, prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar, @@ -1770,6 +1775,7 @@ def completion_cost( service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, + billed_token_rates=_billed_token_rates, ) return _final_cost diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ca2cca5360f..a9f3c0e9325 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -201,6 +201,7 @@ if TYPE_CHECKING: from mcp.types import EmbeddedResource, ImageContent, TextContent from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( @@ -581,6 +582,7 @@ class Logging(LiteLLMLoggingBaseClass): # Initialize cost breakdown field self.cost_breakdown: CostBreakdown | None = None + self.billed_token_rates: BilledTokenRates | None = None # Init Caching related details self.caching_details: CachingDetails | None = None @@ -1585,6 +1587,7 @@ class Logging(LiteLLMLoggingBaseClass): service_tier: str | None = None, data_residency: str | None = None, vertex_location: str | None = None, + billed_token_rates: "BilledTokenRates | None" = None, ) -> None: """ Helper method to store cost breakdown in the logging object. @@ -1604,8 +1607,10 @@ class Logging(LiteLLMLoggingBaseClass): service_tier: Tier the costs above were priced on, already resolved data_residency: Region uplift the costs above were priced on, already resolved vertex_location: Vertex AI location the costs above were priced on, already resolved + billed_token_rates: Per-token rates the costs above were billed at, already resolved """ + self.billed_token_rates = billed_token_rates self.cost_breakdown = CostBreakdown( input_cost=input_cost, output_cost=output_cost, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e2168528e6b..42f88bbb425 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1302,34 +1302,6 @@ def _coerce_token_count(value: object) -> int: return value if isinstance(value, int) and value > 0 else 0 -@dataclass(frozen=True, slots=True) -class TokenTypeCostBreakdown: - reasoning_cost: float - cache_read_cost: float - cache_creation_cost: float - - -def _reasoning_token_count(usage: Usage) -> int: - parsed: Final = ( - parse_completion_tokens_details(usage)["reasoning_tokens"] if usage.completion_tokens_details is not None else 0 - ) - return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0)) - - -def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]: - """(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details - first, then the private top-level counters the Usage constructor mirrors cache tokens onto for - providers/callers that bypass the details.""" - parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None - parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0 - parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0 - return ( - parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)), - parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)), - parsed["cache_creation_token_details"] if parsed is not None else None, - ) - - @dataclass(frozen=True, slots=True) class BilledTokenRates: """Per-token rates one request's usage bills at, after token tiers, off-peak windows and the @@ -1355,6 +1327,37 @@ class BilledTokenRates: ) +@dataclass(frozen=True, slots=True) +class TokenTypeCostBreakdown: + reasoning_cost: float + cache_read_cost: float + cache_creation_cost: float + rates: BilledTokenRates | None = None + """Rates these lines were billed at, so a caller reporting both cannot resolve them a second, + differently-argued way. None when the model's pricing could not be resolved.""" + + +def _reasoning_token_count(usage: Usage) -> int: + parsed: Final = ( + parse_completion_tokens_details(usage)["reasoning_tokens"] if usage.completion_tokens_details is not None else 0 + ) + return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0)) + + +def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]: + """(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details + first, then the private top-level counters the Usage constructor mirrors cache tokens onto for + providers/callers that bypass the details.""" + parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None + parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0 + parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0 + return ( + parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)), + parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)), + parsed["cache_creation_token_details"] if parsed is not None else None, + ) + + def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRates: """Flat custom pricing has no tiers, uplifts or reasoning rate: cache tokens bill at the configured cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does.""" @@ -1497,6 +1500,7 @@ def get_token_type_cost_breakdown( reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, cache_read_cost=float(cache_read_tokens) * rates.cache_read_input_token_cost, cache_creation_cost=cache_creation_cost, + rates=rates, ) diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 1faa66584d5..493f75008c3 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -21,7 +21,6 @@ import litellm from litellm._internal_context import current_billing_time, pinned_billing_time from litellm._logging import verbose_proxy_logger from litellm.cost_calculator import completion_cost -from litellm.litellm_core_utils.llm_cost_calc.utils import get_billed_token_rates from litellm.proxy._types import ( CommonProxyErrors, CostEstimateRequest, @@ -85,9 +84,9 @@ def _extract_custom_pricing( ) -def _lookup_model_info(model: str) -> ModelInfo | None: +def _lookup_model_info(model: str, custom_llm_provider: str | None = None) -> ModelInfo | None: try: - return litellm.get_model_info(model=model) + return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) except Exception: return None @@ -122,7 +121,7 @@ def _resolve_model_for_cost_lookup(model: str) -> ResolvedCostModel: if resolved_model: verbose_proxy_logger.debug("Resolved model '%s' to '%s' from router", model, resolved_model) custom_cost_per_token: Final = _extract_custom_pricing( - litellm_params, model_info, _lookup_model_info(str(resolved_model)) + litellm_params, model_info, _lookup_model_info(str(resolved_model), provider) ) return ResolvedCostModel(str(resolved_model), provider, custom_cost_per_token) except Exception as e: @@ -632,10 +631,9 @@ async def estimate_cost( function_id="cost-estimate", ) - # The totals, the per-token-type lines and the reported rates each resolve pricing on their - # own path. Pinning one moment keeps an off-peak window that opens mid-quote from splitting them. - billed_at: Final = current_billing_time() - with pinned_billing_time(billed_at): + # Pinning one moment keeps an off-peak window that opens mid-quote from pricing the totals on + # one side of it and the reported rates on the other. + with pinned_billing_time(current_billing_time()): # Use completion_cost which handles all the logic including margins/discounts try: cost_per_request: Final = completion_cost( @@ -653,19 +651,16 @@ async def estimate_cost( }, ) - rates: Final = get_billed_token_rates( - model=resolved_model, - custom_llm_provider=resolved_provider, - usage=usage, - custom_cost_per_token=resolved.custom_cost_per_token, - current_time=billed_at, - ) - + # The rates come back from the pricing call itself rather than a second lookup, so they are the + # ones the cost lines above billed at even when completion_cost infers a provider this endpoint + # never resolved (an unrouted "xai/grok-4" prices on xai's inclusive tier thresholds; a lookup + # here without that provider would report the sub-200k rate for a line billed above it). + rates: Final = litellm_logging_obj.billed_token_rates per_request: Final = _cost_lines(cost_per_request, litellm_logging_obj.cost_breakdown) daily: Final = per_request.times(request.num_requests_per_day) monthly: Final = per_request.times(request.num_requests_per_month) - model_info: Final = _lookup_model_info(resolved_model) + model_info: Final = _lookup_model_info(resolved_model, resolved_provider) mapped_provider: Final = model_info.get("litellm_provider") if model_info is not None else None custom_llm_provider: Final = mapped_provider if mapped_provider is not None else resolved_provider diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 90178428018..5a19ed0277a 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -33,7 +33,6 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( CostCalculatorUtils, PromptTokensDetailsResult, TokenRates, - TokenTypeCostBreakdown, _calculate_input_cost, _get_token_base_cost, _is_off_peak, @@ -4023,6 +4022,49 @@ def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_m assert peak_completion_cost == pytest.approx(500 * peak_rates.output_cost_per_token) +def test_the_token_type_breakdown_carries_the_rates_it_billed_at(monkeypatch): + """Callers that report both the lines and the rates read the rates off the breakdown rather than + resolving them a second time, so the breakdown has to hand back exactly what it billed at.""" + monkeypatch.setitem( + litellm.model_cost, + "xai/tiered-model", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "input_cost_per_token_above_200k_tokens": 6e-6, + "output_cost_per_token_above_200k_tokens": 3e-5, + "cache_read_input_token_cost_above_200k_tokens": 6e-7, + "litellm_provider": "xai", + "mode": "chat", + }, + ) + usage = Usage( + prompt_tokens=200_000, + completion_tokens=1_000, + total_tokens=201_000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100_000), + ) + + breakdown = get_token_type_cost_breakdown(model="xai/tiered-model", custom_llm_provider="xai", usage=usage) + + assert breakdown.rates == get_billed_token_rates( + model="xai/tiered-model", custom_llm_provider="xai", usage=usage + ) + assert breakdown.rates.cache_read_input_token_cost == pytest.approx(6e-7) + assert breakdown.cache_read_cost == pytest.approx(100_000 * breakdown.rates.cache_read_input_token_cost) + + +def test_the_token_type_breakdown_reports_no_rates_for_an_unpriced_model(): + usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + + breakdown = get_token_type_cost_breakdown( + model="no-such-model-anywhere", custom_llm_provider="openai", usage=usage + ) + + assert breakdown.rates is None + + def test_billed_token_rates_are_none_for_an_unpriced_model(): usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) @@ -4036,9 +4078,7 @@ def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost model="gpt-4o", custom_llm_provider="openai", usage=usage ) - assert breakdown == TokenTypeCostBreakdown( - reasoning_cost=0.0, cache_read_cost=0.0, cache_creation_cost=0.0 - ) + assert (breakdown.reasoning_cost, breakdown.cache_read_cost, breakdown.cache_creation_cost) == (0.0, 0.0, 0.0) @pytest.mark.parametrize( @@ -4110,9 +4150,7 @@ def test_token_type_cost_breakdown_handles_unknown_model_gracefully(): completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=5), ), ) - assert breakdown == TokenTypeCostBreakdown( - reasoning_cost=0.0, cache_read_cost=0.0, cache_creation_cost=0.0 - ) + assert (breakdown.reasoning_cost, breakdown.cache_read_cost, breakdown.cache_creation_cost) == (0.0, 0.0, 0.0) def test_token_type_cost_breakdown_applies_regional_uplift(_local_model_cost_map): diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index e9485f3a044..7ece35ceedf 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -1139,6 +1139,45 @@ class TestEstimateCostCacheAndReasoningTokens: assert response.output_cost_per_request == pytest.approx(OUTPUT_TOKENS * response.output_cost_per_token) + @pytest.mark.asyncio + async def test_an_unrouted_model_reports_the_rates_of_the_provider_the_calculator_inferred(self, monkeypatch): + """The cost calculator infers a provider this endpoint never resolved, and the provider decides + whether a tier threshold is inclusive. xai bills a request sitting exactly on the 200k threshold + at the tier rate, so the reported rates have to be the tier's rather than the sub-tier base.""" + an_xai_model = "xai/tiered-model" + monkeypatch.setitem( + litellm.model_cost, + an_xai_model, + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "input_cost_per_token_above_200k_tokens": 6e-6, + "output_cost_per_token_above_200k_tokens": 3e-5, + "cache_read_input_token_cost_above_200k_tokens": 6e-7, + "litellm_provider": "xai", + "mode": "chat", + }, + ) + + response = await _estimate( + None, + model=an_xai_model, + input_tokens=200_000, + cache_read_input_tokens=100_000, + output_tokens=1_000, + ) + + assert response.input_cost_per_token == pytest.approx(6e-6) + assert response.output_cost_per_token == pytest.approx(3e-5) + assert response.cache_read_input_token_cost == pytest.approx(6e-7) + assert response.cache_read_cost_per_request == pytest.approx(100_000 * response.cache_read_input_token_cost) + assert response.input_cost_per_request == pytest.approx( + 100_000 * response.input_cost_per_token + response.cache_read_cost_per_request + ) + assert response.output_cost_per_request == pytest.approx(1_000 * response.output_cost_per_token) + + class TestCostEstimateRequestTokenSubsets: def test_cache_tokens_beyond_the_input_tokens_are_rejected(self): with pytest.raises(ValidationError, match="cannot exceed input_tokens"): diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 910587d3dc5..8f8a7640c08 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3736,6 +3736,62 @@ def test_completion_cost_logs_reasoning_and_cache_breakdown(_local_model_cost_ma assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(100 * 3e-08) +def test_completion_cost_logs_the_rates_it_billed_at(monkeypatch): + """A caller reporting the cost lines beside their per-token rates reads both off this one call. + completion_cost infers the provider, and xai's inclusive tier thresholds put a request sitting + exactly on 200k at the tier rate, which a lookup made without that inferred provider would miss. + """ + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging + + monkeypatch.setitem( + litellm.model_cost, + "xai/tiered-model", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "input_cost_per_token_above_200k_tokens": 6e-6, + "output_cost_per_token_above_200k_tokens": 3e-5, + "cache_read_input_token_cost_above_200k_tokens": 6e-7, + "litellm_provider": "xai", + "mode": "chat", + }, + ) + logging_obj = Logging( + model="xai/tiered-model", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="billed-rates", + function_id="f", + ) + usage = Usage( + prompt_tokens=200_000, + completion_tokens=1_000, + total_tokens=201_000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100_000), + ) + + litellm.completion_cost( + completion_response=ModelResponse(model="xai/tiered-model", usage=usage), + model="xai/tiered-model", + custom_llm_provider=None, + litellm_logging_obj=logging_obj, + ) + + rates = logging_obj.billed_token_rates + assert rates is not None + assert rates.input_cost_per_token == pytest.approx(6e-6) + assert rates.cache_read_input_token_cost == pytest.approx(6e-7) + assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx( + 100_000 * rates.cache_read_input_token_cost + ) + assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(1_000 * rates.output_cost_per_token) + + def test_completion_cost_logs_cache_and_reasoning_breakdown_for_custom_pricing(): """ A custom-priced deployment bills cache tokens at its custom cache rates, but the From 2ece8735384e063b88ca91cf9d2f8f6d2e94b30a Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 17:05:38 +0000 Subject: [PATCH 193/411] test(e2e): lite CLI never fetches the model cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/coverage_registry/other.yaml | 2 + tests/e2e/other/test_cli_cost_map_e2e.py | 109 +++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 tests/e2e/other/test_cli_cost_map_e2e.py diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 814ebae2e0b..8ac4fc0762c 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -2,6 +2,8 @@ # PROMOTION NOTE: the auth cluster (~14 cells) is a candidate to promote to its own module once stable. - {id: other.auth.master_key.valid_allows, module: other, tier: P0, area: auth, assertions: [valid_allows], source: "user_api_key_auth.py:1569-1588", rationale: "Master key authenticates; timing-safe compare"} - {id: other.auth.master_key.invalid_denied, module: other, tier: P0, area: auth, assertions: [invalid_denied], source: "user_api_key_auth.py:1580", rationale: "Invalid master key rejected"} +- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} +- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} - {id: other.auth.llm_chat.missing_header_denied, module: other, tier: P0, area: auth, assertions: [missing_header_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Chat with no Authorization header is 401/403"} - {id: other.auth.llm_chat.invalid_bearer_denied, module: other, tier: P0, area: auth, assertions: [invalid_bearer_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Bearer invalid_token on chat is 401/403"} - {id: other.auth.llm_chat.no_bearer_prefix_denied, module: other, tier: P0, area: auth, assertions: [no_bearer_prefix_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Token without Bearer scheme on chat is 401/403"} diff --git a/tests/e2e/other/test_cli_cost_map_e2e.py b/tests/e2e/other/test_cli_cost_map_e2e.py new file mode 100644 index 00000000000..6022d5a0a4c --- /dev/null +++ b/tests/e2e/other/test_cli_cost_map_e2e.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import threading +from collections.abc import Mapping +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import MASTER_KEY, PROXY_BASE_URL +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + + +def _start_cost_map_server(request_log: Path) -> tuple[ThreadingHTTPServer, threading.Thread]: + class CostMapHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + with request_log.open("a", encoding="utf-8") as log_file: + log_file.write(f"{self.path}\n") + body: Final = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), CostMapHandler) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread + + +def _run_lite( + args: tuple[str, ...], + server: ThreadingHTTPServer, + env: Mapping[str, str], +) -> subprocess.CompletedProcess[str]: + lite_path: Final = shutil.which("lite") + assert lite_path is not None, "the installed lite executable is required for e2e coverage" + try: + return subprocess.run( + [lite_path, *args], + capture_output=True, + text=True, + timeout=60, + env=env, + ) + finally: + server.shutdown() + server.server_close() + + +def _request_count(request_log: Path) -> int: + return request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 + + +class TestLiteCliCostMapFetch: + @pytest.mark.covers("other.cli.model_cost_map.version_skips_fetch") + def test_lite_version_makes_no_cost_map_request(self, tmp_path: Path) -> None: + request_log: Final = tmp_path / "requests.log" + server, thread = _start_cost_map_server(request_log) + source_root: Final = str(Path(__file__).resolve().parents[3]) + pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) + env: Final = { + **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, + "PYTHONPATH": pythonpath, + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": PROXY_BASE_URL, + } + try: + result: Final = _run_lite(("--version",), server, env) + finally: + thread.join(timeout=10) + + assert result.returncode == 0 + assert "LiteLLM Proxy CLI Version" in result.stdout + assert _request_count(request_log) == 0 + + @pytest.mark.covers("other.cli.model_cost_map.models_list_skips_fetch") + def test_lite_models_list_uses_proxy_not_cost_map(self, tmp_path: Path, proxy: ProxyClient) -> None: + model_names: Final = tuple(entry.model_name for entry in proxy.model_info()) + assert model_names + request_log: Final = tmp_path / "requests.log" + server, thread = _start_cost_map_server(request_log) + source_root: Final = str(Path(__file__).resolve().parents[3]) + pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) + env: Final = { + **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, + "PYTHONPATH": pythonpath, + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": PROXY_BASE_URL, + "LITELLM_PROXY_API_KEY": MASTER_KEY, + } + try: + result: Final = _run_lite(("models", "list"), server, env) + finally: + thread.join(timeout=10) + + assert result.returncode == 0 + assert result.stdout.strip() + assert any(model_name in result.stdout for model_name in model_names) + assert _request_count(request_log) == 0 From 6e71b90a888b91fe3aa281fa14da58f258b9f056 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 17:07:34 +0000 Subject: [PATCH 194/411] test(e2e): dedupe lite env setup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/coverage_registry/other.yaml | 4 ++-- tests/e2e/other/test_cli_cost_map_e2e.py | 29 ++++++++++-------------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 8ac4fc0762c..a7ba0dbb8cb 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -2,8 +2,6 @@ # PROMOTION NOTE: the auth cluster (~14 cells) is a candidate to promote to its own module once stable. - {id: other.auth.master_key.valid_allows, module: other, tier: P0, area: auth, assertions: [valid_allows], source: "user_api_key_auth.py:1569-1588", rationale: "Master key authenticates; timing-safe compare"} - {id: other.auth.master_key.invalid_denied, module: other, tier: P0, area: auth, assertions: [invalid_denied], source: "user_api_key_auth.py:1580", rationale: "Invalid master key rejected"} -- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} -- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} - {id: other.auth.llm_chat.missing_header_denied, module: other, tier: P0, area: auth, assertions: [missing_header_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Chat with no Authorization header is 401/403"} - {id: other.auth.llm_chat.invalid_bearer_denied, module: other, tier: P0, area: auth, assertions: [invalid_bearer_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Bearer invalid_token on chat is 401/403"} - {id: other.auth.llm_chat.no_bearer_prefix_denied, module: other, tier: P0, area: auth, assertions: [no_bearer_prefix_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Token without Bearer scheme on chat is 401/403"} @@ -50,3 +48,5 @@ - {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"} - {id: other.a2a.version.serves_pinned_0_3, module: other, tier: P1, area: a2a, assertions: [serves_pinned_0_3], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 0.3 returns the flat 0.3 message shape (parts on the result)"} - {id: other.a2a.version.serves_pinned_1_0, module: other, tier: P1, area: a2a, assertions: [serves_pinned_1_0], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 1.0 returns the nested 1.0 message shape (result.message with ROLE_AGENT)"} +- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} +- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} diff --git a/tests/e2e/other/test_cli_cost_map_e2e.py b/tests/e2e/other/test_cli_cost_map_e2e.py index 6022d5a0a4c..62cd3d6c3f5 100644 --- a/tests/e2e/other/test_cli_cost_map_e2e.py +++ b/tests/e2e/other/test_cli_cost_map_e2e.py @@ -37,6 +37,16 @@ def _start_cost_map_server(request_log: Path) -> tuple[ThreadingHTTPServer, thre return server, thread +def _lite_env(server: ThreadingHTTPServer, api_key: str | None) -> dict[str, str]: + base_env: Final = {key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"} + return { + **base_env, + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": PROXY_BASE_URL, + **({"LITELLM_PROXY_API_KEY": api_key} if api_key is not None else {}), + } + + def _run_lite( args: tuple[str, ...], server: ThreadingHTTPServer, @@ -66,14 +76,7 @@ class TestLiteCliCostMapFetch: def test_lite_version_makes_no_cost_map_request(self, tmp_path: Path) -> None: request_log: Final = tmp_path / "requests.log" server, thread = _start_cost_map_server(request_log) - source_root: Final = str(Path(__file__).resolve().parents[3]) - pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) - env: Final = { - **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, - "PYTHONPATH": pythonpath, - "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", - "LITELLM_PROXY_URL": PROXY_BASE_URL, - } + env: Final = _lite_env(server, None) try: result: Final = _run_lite(("--version",), server, env) finally: @@ -89,15 +92,7 @@ class TestLiteCliCostMapFetch: assert model_names request_log: Final = tmp_path / "requests.log" server, thread = _start_cost_map_server(request_log) - source_root: Final = str(Path(__file__).resolve().parents[3]) - pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) - env: Final = { - **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, - "PYTHONPATH": pythonpath, - "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", - "LITELLM_PROXY_URL": PROXY_BASE_URL, - "LITELLM_PROXY_API_KEY": MASTER_KEY, - } + env: Final = _lite_env(server, MASTER_KEY) try: result: Final = _run_lite(("models", "list"), server, env) finally: From 996ee5635ab79854992661279c2e8db793b3dda4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:35:50 -0700 Subject: [PATCH 195/411] perf(proxy): pipeline spend counter increments into one Redis call per request (#40371) * perf(proxy): pipeline spend counter increments into one redis call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): apply surviving spend increments before raising scope error Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(proxy): ruff format spend counter helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): settle inner spend counter gathers and fall back per key on pipeline failure Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): suppress BLE001 on pipeline fallback catch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): invalidate all batched spend counters on pipeline failure Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 331 +++++++---- .../proxy/proxy_server/test_spend_counters.py | 538 ++++++++++-------- .../test_budget_reservation_redis_failure.py | 9 + .../proxy/test_budget_reservation.py | 9 + tests/test_litellm/proxy/test_proxy_server.py | 94 ++- 5 files changed, 609 insertions(+), 372 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7aed0553b4b..c241e66049b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -17,6 +17,7 @@ import time import traceback import warnings from collections.abc import AsyncGenerator, AsyncIterator, Callable, Collection, Mapping, MutableMapping, Sequence +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from types import MappingProxyType, UnionType from typing import ( @@ -131,6 +132,7 @@ from litellm.router_utils.auto_router_tuning_baseline import ( snapshot_tuning_baselines, tuning_limit_violation, ) +from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.utils import ( ModelResponse, ModelResponseStream, @@ -2703,6 +2705,12 @@ async def _read_spend_counter_estimate(counter_key: str, fallback_spend: float) return fallback_spend, False +@dataclass(frozen=True, slots=True) +class _PendingSpendIncrement: + counter_key: str + increment: float + + async def increment_spend_counters( token: str | None, team_id: str | None, @@ -2737,7 +2745,7 @@ async def increment_spend_counters( cost: Final[float] = response_cost - async def _key_scope(key_token: str) -> None: + async def _key_scope(key_token: str) -> tuple[_PendingSpendIncrement | BaseException, ...]: # key_token arrives pre-hashed from metadata["user_api_key"] (auth flow # hashes raw "sk-..." keys before they reach the callback). The # startswith("sk-") check is a safety net matching update_cache — @@ -2748,30 +2756,29 @@ async def increment_spend_counters( hash_token(token=key_token) if isinstance(key_token, str) and key_token.startswith("sk-") else key_token ) key_counter_key: Final = f"spend:key:{hashed_token}" - if key_counter_key not in reserved_counter_keys: - await _init_and_increment_spend_counter( - counter_key=key_counter_key, - source_cache_key=hashed_token, - increment=cost, + key_pending: Final[tuple[_PendingSpendIncrement, ...]] = ( + () + if key_counter_key in reserved_counter_keys + else ( + await _prepare_spend_counter_increment( + counter_key=key_counter_key, + source_cache_key=hashed_token, + increment=cost, + ), ) - - key_obj: Final[object] = await user_api_key_cache.async_get_cache(key=hashed_token) - if key_obj is None: - return - key_budget_limits = getattr(key_obj, "budget_limits", None) or ( - key_obj.get("budget_limits") if isinstance(key_obj, dict) else None ) - if isinstance(key_budget_limits, str): - key_budget_limits = json.loads(key_budget_limits) - if not isinstance(key_budget_limits, list): - return - for window in key_budget_limits: - duration = window["budget_duration"] if isinstance(window, dict) else window.budget_duration - key_window_reset_at = window.get("reset_at") if isinstance(window, dict) else window.reset_at - key_window_counter = f"spend:key:{hashed_token}:window:{duration}" + + async def _key_window_increment(window: object) -> _PendingSpendIncrement | None: + duration = ( + window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None) + ) + key_window_reset_at = ( + window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None) + ) + key_window_counter: Final = f"spend:key:{hashed_token}:window:{duration}" key_window_start = get_budget_window_start(window) - if key_window_counter not in reserved_counter_keys: - await _init_and_increment_window_spend_counter( + pending_window: Final = ( + await _prepare_window_spend_counter_increment( counter_key=key_window_counter, entity_type="Key", entity_id=hashed_token, @@ -2779,6 +2786,9 @@ async def increment_spend_counters( window_start=key_window_start, increment=cost, ) + if key_window_counter not in reserved_counter_keys + else None + ) await _enqueue_window_spend_row_update( entity_type=Litellm_EntityType.KEY, entity_id=hashed_token, @@ -2788,33 +2798,48 @@ async def increment_spend_counters( increment=cost, request_started_at=request_started_at, ) + return pending_window - async def _team_scope(scope_team_id: str) -> None: - team_counter_key: Final = f"spend:team:{scope_team_id}" - if team_counter_key not in reserved_counter_keys: - await _init_and_increment_spend_counter( - counter_key=team_counter_key, - source_cache_key=f"team_id:{scope_team_id}", - increment=cost, - ) - - team_obj: Final[object] = await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}") - if team_obj is None: - return - team_budget_limits = getattr(team_obj, "budget_limits", None) or ( - team_obj.get("budget_limits") if isinstance(team_obj, dict) else None + key_obj: Final[object] = await user_api_key_cache.async_get_cache(key=hashed_token) + if key_obj is None: + return key_pending + key_budget_limits = getattr(key_obj, "budget_limits", None) or ( + key_obj.get("budget_limits") if isinstance(key_obj, dict) else None ) - if isinstance(team_budget_limits, str): - team_budget_limits = json.loads(team_budget_limits) - if not isinstance(team_budget_limits, list): - return - for window in team_budget_limits: - duration = window["budget_duration"] if isinstance(window, dict) else window.budget_duration - team_window_reset_at = window.get("reset_at") if isinstance(window, dict) else window.reset_at - team_window_counter = f"spend:team:{scope_team_id}:window:{duration}" + if isinstance(key_budget_limits, str): + key_budget_limits = json.loads(key_budget_limits) + if not isinstance(key_budget_limits, list): + return key_pending + window_pending: Final = await asyncio.gather( + *(_key_window_increment(window) for window in key_budget_limits), return_exceptions=True + ) + return key_pending + tuple(item for item in window_pending if item is not None) + + async def _team_scope(scope_team_id: str) -> tuple[_PendingSpendIncrement | BaseException, ...]: + team_counter_key: Final = f"spend:team:{scope_team_id}" + team_pending: Final[tuple[_PendingSpendIncrement, ...]] = ( + () + if team_counter_key in reserved_counter_keys + else ( + await _prepare_spend_counter_increment( + counter_key=team_counter_key, + source_cache_key=f"team_id:{scope_team_id}", + increment=cost, + ), + ) + ) + + async def _team_window_increment(window: object) -> _PendingSpendIncrement | None: + duration = ( + window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None) + ) + team_window_reset_at = ( + window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None) + ) + team_window_counter: Final = f"spend:team:{scope_team_id}:window:{duration}" team_window_start = get_budget_window_start(window) - if team_window_counter not in reserved_counter_keys: - await _init_and_increment_window_spend_counter( + pending_window: Final = ( + await _prepare_window_spend_counter_increment( counter_key=team_window_counter, entity_type="Team", entity_id=scope_team_id, @@ -2822,6 +2847,9 @@ async def increment_spend_counters( window_start=team_window_start, increment=cost, ) + if team_window_counter not in reserved_counter_keys + else None + ) await _enqueue_window_spend_row_update( entity_type=Litellm_EntityType.TEAM, entity_id=scope_team_id, @@ -2831,25 +2859,47 @@ async def increment_spend_counters( increment=cost, request_started_at=request_started_at, ) + return pending_window - async def _team_member_scope(scope_user_id: str, scope_team_id: str) -> None: + team_obj: Final[object] = await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}") + if team_obj is None: + return team_pending + team_budget_limits = getattr(team_obj, "budget_limits", None) or ( + team_obj.get("budget_limits") if isinstance(team_obj, dict) else None + ) + if isinstance(team_budget_limits, str): + team_budget_limits = json.loads(team_budget_limits) + if not isinstance(team_budget_limits, list): + return team_pending + window_pending: Final = await asyncio.gather( + *(_team_window_increment(window) for window in team_budget_limits), return_exceptions=True + ) + return team_pending + tuple(item for item in window_pending if item is not None) + + async def _team_member_scope( + scope_user_id: str, scope_team_id: str + ) -> tuple[_PendingSpendIncrement | BaseException, ...]: team_member_counter_key: Final = f"spend:team_member:{scope_user_id}:{scope_team_id}" if team_member_counter_key in reserved_counter_keys: - return - await _init_and_increment_spend_counter( - counter_key=team_member_counter_key, - source_cache_key=f"team_membership:{scope_user_id}:{scope_team_id}", - increment=cost, + return () + return ( + await _prepare_spend_counter_increment( + counter_key=team_member_counter_key, + source_cache_key=f"team_membership:{scope_user_id}:{scope_team_id}", + increment=cost, + ), ) - async def _user_scope(scope_user_id: str) -> None: + async def _user_scope(scope_user_id: str) -> tuple[_PendingSpendIncrement | BaseException, ...]: user_counter_key: Final = f"spend:user:{scope_user_id}" if user_counter_key in reserved_counter_keys: - return - await _init_and_increment_spend_counter( - counter_key=user_counter_key, - source_cache_key=scope_user_id, - increment=cost, + return () + return ( + await _prepare_spend_counter_increment( + counter_key=user_counter_key, + source_cache_key=scope_user_id, + increment=cost, + ), ) scope_coros: Final = tuple( @@ -2859,7 +2909,7 @@ async def increment_spend_counters( _team_scope(team_id) if team_id is not None else None, _team_member_scope(user_id, team_id) if user_id is not None and team_id is not None else None, _user_scope(user_id) if user_id is not None else None, - _increment_end_user_and_tag_spend_counters( + _prepare_end_user_and_tag_spend_increments( end_user_id=end_user_id, tags=tags, response_cost=cost, @@ -2867,14 +2917,14 @@ async def increment_spend_counters( ) if end_user_id is not None or tags is not None else None, - _increment_model_access_group_spend_counters( + _prepare_model_access_group_spend_increments( model_access_groups=model_access_groups, response_cost=cost, reserved_counter_keys=reserved_counter_keys, ) if model_access_groups else None, - _increment_org_spend_counter( + _prepare_org_spend_increment( org_id=org_id, response_cost=cost, reserved_counter_keys=reserved_counter_keys, @@ -2889,7 +2939,20 @@ async def increment_spend_counters( # as orphaned tasks that race the caller's reservation-counter invalidation; # all scopes settle, then the first error propagates as before. scope_results: Final = await asyncio.gather(*scope_coros, return_exceptions=True) - scope_errors: Final = [r for r in scope_results if isinstance(r, BaseException)] + scope_errors: Final = tuple( + item + for scope in scope_results + for item in (scope if isinstance(scope, tuple) else (scope,)) + if isinstance(item, BaseException) + ) + pending: Final = tuple( + item + for scope in scope_results + if not isinstance(scope, BaseException) + for item in scope + if not isinstance(item, BaseException) + ) + await _apply_spend_counter_increments(pending=pending) if scope_errors: raise scope_errors[0] @@ -2932,41 +2995,49 @@ async def _reconcile_budget_reservation_for_counter_update( return reserved_counter_keys -async def _increment_end_user_and_tag_spend_counters( +async def _prepare_end_user_and_tag_spend_increments( end_user_id: str | None, tags: list[str] | None, response_cost: float, reserved_counter_keys: set[str], -) -> None: - if end_user_id is not None: - await _init_and_increment_unreserved_spend_counter( - counter_key=f"spend:end_user:{end_user_id}", - source_cache_key=end_user_cache_key(end_user_id), - increment=response_cost, - reserved_counter_keys=reserved_counter_keys, - ) - - if tags is None: - return - - seen_tags: Final[set[str]] = set() - for tag_name in tags: - if not tag_name or not isinstance(tag_name, str) or tag_name in seen_tags: - continue - seen_tags.add(tag_name) - await _init_and_increment_unreserved_spend_counter( - counter_key=f"spend:tag:{tag_name}", - source_cache_key=tag_cache_key(tag_name), - increment=response_cost, - reserved_counter_keys=reserved_counter_keys, - ) +) -> tuple[_PendingSpendIncrement | BaseException, ...]: + unique_tags: Final = ( + tuple(dict.fromkeys(tag for tag in tags if tag and isinstance(tag, str))) if tags is not None else () + ) + results: Final = await asyncio.gather( + *( + coro + for coro in ( + _prepare_unreserved_spend_counter_increment( + counter_key=f"spend:end_user:{end_user_id}", + source_cache_key=end_user_cache_key(end_user_id), + increment=response_cost, + reserved_counter_keys=reserved_counter_keys, + ) + if end_user_id is not None + else None, + *( + _prepare_unreserved_spend_counter_increment( + counter_key=f"spend:tag:{tag_name}", + source_cache_key=tag_cache_key(tag_name), + increment=response_cost, + reserved_counter_keys=reserved_counter_keys, + ) + for tag_name in unique_tags + ), + ) + if coro is not None + ), + return_exceptions=True, + ) + return tuple(item for item in results if item is not None) -async def _increment_model_access_group_spend_counters( +async def _prepare_model_access_group_spend_increments( model_access_groups: Sequence[object], response_cost: float, reserved_counter_keys: set[str], -) -> None: +) -> tuple[_PendingSpendIncrement | BaseException, ...]: """Charge the model access groups that authorized this request. Without this the counter auth reads is written only by the reservation path, so @@ -2980,55 +3051,63 @@ async def _increment_model_access_group_spend_counters( unique_groups: Final = tuple( dict.fromkeys(group for group in model_access_groups if group and isinstance(group, str)) ) - for group in unique_groups: - await _init_and_increment_unreserved_spend_counter( - counter_key=model_access_group_spend_counter_key(group), - source_cache_key=model_access_group_cache_key(group), - increment=response_cost, - reserved_counter_keys=reserved_counter_keys, - ) + results: Final = await asyncio.gather( + *( + _prepare_unreserved_spend_counter_increment( + counter_key=model_access_group_spend_counter_key(group), + source_cache_key=model_access_group_cache_key(group), + increment=response_cost, + reserved_counter_keys=reserved_counter_keys, + ) + for group in unique_groups + ), + return_exceptions=True, + ) + return tuple(item for item in results if item is not None) -async def _increment_org_spend_counter( +async def _prepare_org_spend_increment( org_id: str | None, response_cost: float, reserved_counter_keys: set[str], -) -> None: +) -> tuple[_PendingSpendIncrement, ...]: if org_id is None: - return + return () - await _init_and_increment_unreserved_spend_counter( + pending: Final = await _prepare_unreserved_spend_counter_increment( counter_key=f"spend:org:{org_id}", source_cache_key=[f"org_id:{org_id}:with_budget", f"org_id:{org_id}"], increment=response_cost, reserved_counter_keys=reserved_counter_keys, ) + return (pending,) if pending is not None else () -async def _init_and_increment_unreserved_spend_counter( +async def _prepare_unreserved_spend_counter_increment( counter_key: str, source_cache_key: str | list[str], increment: float, reserved_counter_keys: set[str], -) -> None: +) -> _PendingSpendIncrement | None: if counter_key in reserved_counter_keys: - return + return None - await _init_and_increment_spend_counter( + return await _prepare_spend_counter_increment( counter_key=counter_key, source_cache_key=source_cache_key, increment=increment, ) -async def _init_and_increment_spend_counter( +async def _prepare_spend_counter_increment( counter_key: str, source_cache_key: str | list[str], increment: float, -): +) -> _PendingSpendIncrement: """ Initialize counter from the authoritative DB spend value if not yet - set, then atomically increment in both in-memory and Redis. + set, then return the pending increment for the caller to apply in one + pipelined Redis call. On first access per pod: 1. Check spend_counter_cache (in-memory -> Redis via DualCache) @@ -3040,13 +3119,13 @@ async def _init_and_increment_spend_counter( the counter as absent and seed it. Using increment means the worst case is over-counting (conservative, blocks slightly early) rather than under-counting (would allow overspend). - 4. Increment atomically (both in-memory + Redis) + 4. Increment is returned for the caller to apply via pipeline """ await _ensure_spend_counter_initialized( counter_key=counter_key, source_cache_key=source_cache_key, ) - await _increment_spend_counter_cache(counter_key=counter_key, increment=increment) + return _PendingSpendIncrement(counter_key=counter_key, increment=increment) async def _enqueue_window_spend_row_update( @@ -3098,20 +3177,20 @@ async def _enqueue_window_spend_row_update( ) -async def _init_and_increment_window_spend_counter( +async def _prepare_window_spend_counter_increment( counter_key: str, entity_type: str, entity_id: str, window_duration: str | None, window_start: datetime | None, increment: float, -): +) -> _PendingSpendIncrement | None: if window_start is None: verbose_proxy_logger.warning( "Skipping spend counter increment for invalid budget window %s", counter_key, ) - return + return None initialized: Final = await _ensure_window_spend_counter_initialized( counter_key=counter_key, @@ -3121,8 +3200,8 @@ async def _init_and_increment_window_spend_counter( window_start=window_start, ) if initialized is False: - return - await _increment_spend_counter_cache(counter_key=counter_key, increment=increment) + return None + return _PendingSpendIncrement(counter_key=counter_key, increment=increment) async def _ensure_spend_counter_initialized( @@ -3255,6 +3334,32 @@ async def _invalidate_spend_counter(counter_key: str): ) +async def _apply_spend_counter_increments(pending: Sequence[_PendingSpendIncrement]) -> None: + if not pending: + return + redis_cache: Final = spend_counter_cache.redis_cache + if redis_cache is None: + for item in pending: + await spend_counter_cache.async_increment_cache( + key=item.counter_key, + value=item.increment, + refresh_ttl=True, + ) + return + ttl: Final = redis_cache.get_ttl() + increment_list: Final = [ # mutable-ok: async_increment_pipeline signature requires list[RedisPipelineIncrementOperation] + RedisPipelineIncrementOperation(key=item.counter_key, increment_value=item.increment, ttl=ttl) + for item in pending + ] + try: + results: Final = await redis_cache.async_increment_pipeline(increment_list=increment_list) + except Exception: + await asyncio.gather(*(_invalidate_spend_counter(counter_key=item.counter_key) for item in pending)) + raise + for item, current_value in zip(pending, results or ()): + spend_counter_cache.in_memory_cache.set_cache(key=item.counter_key, value=current_value) + + async def update_cache( token: str | None, user_id: str | None, diff --git a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py index 86e97a334df..c343652efd9 100644 --- a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py +++ b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py @@ -4,11 +4,12 @@ Pins covered: - ``get_current_spend`` - ``increment_spend_counters`` - ``_reconcile_budget_reservation_for_counter_update`` -- ``_increment_end_user_and_tag_spend_counters`` -- ``_increment_org_spend_counter`` -- ``_init_and_increment_unreserved_spend_counter`` -- ``_init_and_increment_spend_counter`` -- ``_init_and_increment_window_spend_counter`` +- ``_prepare_end_user_and_tag_spend_increments`` +- ``_prepare_org_spend_increment`` +- ``_prepare_unreserved_spend_counter_increment`` +- ``_prepare_spend_counter_increment`` +- ``_prepare_window_spend_counter_increment`` +- ``_apply_spend_counter_increments`` - ``_ensure_spend_counter_initialized`` - ``_get_source_cache_base_spend`` - ``_ensure_window_spend_counter_initialized`` @@ -48,9 +49,7 @@ def _make_spend_counter_cache( cache.in_memory_cache.delete_cache = MagicMock() if with_redis: cache.redis_cache = MagicMock() - cache.redis_cache.async_get_cache = AsyncMock( - return_value=redis_get_value, side_effect=redis_get_side_effect - ) + cache.redis_cache.async_get_cache = AsyncMock(return_value=redis_get_value, side_effect=redis_get_side_effect) cache.redis_cache.async_increment = AsyncMock( return_value=redis_increment_value, side_effect=redis_increment_side_effect, @@ -58,6 +57,8 @@ def _make_spend_counter_cache( cache.redis_cache.async_delete_cache = AsyncMock() cache.redis_cache.async_set_cache = AsyncMock() cache.redis_cache.async_set_max = AsyncMock() + cache.redis_cache.async_increment_pipeline = AsyncMock(return_value=None) + cache.redis_cache.get_ttl = MagicMock(return_value=None) else: cache.redis_cache = None cache.async_increment_cache = AsyncMock(return_value=redis_increment_value) @@ -70,9 +71,7 @@ def _make_spend_counter_cache( def _make_user_api_key_cache(get_value=None, get_side_effect=None): cache = MagicMock() - cache.async_get_cache = AsyncMock( - return_value=get_value, side_effect=get_side_effect - ) + cache.async_get_cache = AsyncMock(return_value=get_value, side_effect=get_side_effect) cache.async_set_cache_pipeline = AsyncMock() return cache @@ -109,9 +108,7 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(monkeypatch ) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - result = await ps.get_current_spend( - counter_key="spend:key:abc", fallback_spend=99.0 - ) + result = await ps.get_current_spend(counter_key="spend:key:abc", fallback_spend=99.0) assert result == 17.0 @@ -136,9 +133,7 @@ async def test_get_current_spend_floors_stale_low_counter_against_db(monkeypatch # the stale counter is repaired up to the authoritative DB value via a # monotonic set-max so other workers read the corrected total, and a # concurrent increment cannot be clobbered - fake_cache.redis_cache.async_set_max.assert_awaited_once_with( - key="spend:key:abc", value=12.0 - ) + fake_cache.redis_cache.async_set_max.assert_awaited_once_with(key="spend:key:abc", value=12.0) @pytest.mark.asyncio @@ -169,9 +164,7 @@ async def test_get_current_spend_no_floor_without_max_budget(monkeypatch): from_db = AsyncMock(return_value=12.0) monkeypatch.setattr(ps.SpendCounterReseed, "from_db", from_db) - result = await ps.get_current_spend( - counter_key="spend:key:abc", fallback_spend=12.0 - ) + result = await ps.get_current_spend(counter_key="spend:key:abc", fallback_spend=12.0) assert result == 2.0 assert from_db.await_count == 0 @@ -210,12 +203,8 @@ async def test_get_current_spend_floor_caches_db_read(monkeypatch): from_db = AsyncMock(return_value=12.0) monkeypatch.setattr(ps.SpendCounterReseed, "from_db", from_db) - first = await ps.get_current_spend( - counter_key="spend:key:abc", fallback_spend=12.0, max_budget=10.0 - ) - second = await ps.get_current_spend( - counter_key="spend:key:abc", fallback_spend=12.0, max_budget=10.0 - ) + first = await ps.get_current_spend(counter_key="spend:key:abc", fallback_spend=12.0, max_budget=10.0) + second = await ps.get_current_spend(counter_key="spend:key:abc", fallback_spend=12.0, max_budget=10.0) assert first == 12.0 assert second == 12.0 @@ -336,9 +325,7 @@ async def test_get_current_spend_floors_window_against_spend_logs(monkeypatch): assert result == 15.0 assert wfsl.await_count == 1 - fake_cache.redis_cache.async_set_max.assert_awaited_once_with( - key=counter_key, value=15.0 - ) + fake_cache.redis_cache.async_set_max.assert_awaited_once_with(key=counter_key, value=15.0) def _make_window_spend_prisma(row=None, spend_logs_total=0.0): @@ -379,9 +366,7 @@ async def test_get_current_spend_floors_window_against_maintained_row(monkeypatc assert result == 15.0 fake_prisma.db.litellm_spendlogs.group_by.assert_not_awaited() - fake_cache.redis_cache.async_set_max.assert_awaited_once_with( - key=counter_key, value=15.0 - ) + fake_cache.redis_cache.async_set_max.assert_awaited_once_with(key=counter_key, value=15.0) @pytest.mark.asyncio @@ -393,9 +378,7 @@ async def test_get_current_spend_floors_window_against_logs_when_row_stale(monke window_start = datetime(2026, 1, 8, tzinfo=timezone.utc) fake_prisma = _make_window_spend_prisma( - row=SimpleNamespace( - window_start=window_start - timedelta(days=7), spend=999.0 - ), + row=SimpleNamespace(window_start=window_start - timedelta(days=7), spend=999.0), spend_logs_total=15.0, ) fake_cache = _make_spend_counter_cache(redis_get_value=2.0) @@ -423,21 +406,13 @@ async def test_get_current_spend_fail_closed_rejects_when_unverifiable(monkeypat rather than admitted on an unverifiable budget.""" from fastapi import HTTPException - fake_cache = _make_spend_counter_cache( - redis_get_side_effect=RuntimeError("redis down") - ) + fake_cache = _make_spend_counter_cache(redis_get_side_effect=RuntimeError("redis down")) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - monkeypatch.setattr( - ps, "general_settings", {"fail_closed_budget_enforcement": True} - ) - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) - ) + monkeypatch.setattr(ps, "general_settings", {"fail_closed_budget_enforcement": True}) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)) with pytest.raises(HTTPException) as exc: - await ps.get_current_spend( - counter_key="spend:key:abc", fallback_spend=1.0, max_budget=10.0 - ) + await ps.get_current_spend(counter_key="spend:key:abc", fallback_spend=1.0, max_budget=10.0) assert exc.value.status_code == 503 @@ -445,18 +420,12 @@ async def test_get_current_spend_fail_closed_rejects_when_unverifiable(monkeypat async def test_get_current_spend_fail_closed_off_admits_when_unverifiable(monkeypatch): """Default (flag off): an unverifiable read keeps the existing behavior and admits using the cached fallback — no new rejection.""" - fake_cache = _make_spend_counter_cache( - redis_get_side_effect=RuntimeError("redis down") - ) + fake_cache = _make_spend_counter_cache(redis_get_side_effect=RuntimeError("redis down")) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "general_settings", {}) - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) - ) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)) - result = await ps.get_current_spend( - counter_key="spend:key:abc", fallback_spend=1.0, max_budget=10.0 - ) + result = await ps.get_current_spend(counter_key="spend:key:abc", fallback_spend=1.0, max_budget=10.0) assert result == 1.0 @@ -466,13 +435,9 @@ async def test_get_current_spend_fail_closed_admits_when_redis_verified(monkeypa authoritative, so an under-budget request is admitted normally.""" fake_cache = _make_spend_counter_cache(redis_get_value=1.0) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - monkeypatch.setattr( - ps, "general_settings", {"fail_closed_budget_enforcement": True} - ) + monkeypatch.setattr(ps, "general_settings", {"fail_closed_budget_enforcement": True}) - result = await ps.get_current_spend( - counter_key="spend:key:abc", fallback_spend=1.0, max_budget=10.0 - ) + result = await ps.get_current_spend(counter_key="spend:key:abc", fallback_spend=1.0, max_budget=10.0) assert result == 1.0 @@ -481,16 +446,10 @@ async def test_get_current_spend_fail_closed_allows_authoritative_fallback(monke """End-user/tag callers pass fallback_authoritative=True (their spend is loaded fresh from the DB in auth), so fail-closed does not reject them even when the counter path is unreadable.""" - fake_cache = _make_spend_counter_cache( - redis_get_side_effect=RuntimeError("redis down") - ) + fake_cache = _make_spend_counter_cache(redis_get_side_effect=RuntimeError("redis down")) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - monkeypatch.setattr( - ps, "general_settings", {"fail_closed_budget_enforcement": True} - ) - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) - ) + monkeypatch.setattr(ps, "general_settings", {"fail_closed_budget_enforcement": True}) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)) result = await ps.get_current_spend( counter_key="spend:end_user:e1", @@ -508,9 +467,7 @@ async def test_get_current_spend_strict_floors_when_fallback_also_stale(monkeypa re-checks the authoritative DB and enforces against it.""" fake_cache = _make_spend_counter_cache(redis_get_value=0.00001) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - monkeypatch.setattr( - ps, "general_settings", {"fail_closed_budget_enforcement": True} - ) + monkeypatch.setattr(ps, "general_settings", {"fail_closed_budget_enforcement": True}) from_db = AsyncMock(return_value=0.5) monkeypatch.setattr(ps.SpendCounterReseed, "from_db", from_db) @@ -532,9 +489,7 @@ async def test_get_current_spend_strict_floors_when_fallback_also_stale(monkeypa @pytest.mark.asyncio async def test_increment_spend_counters_increments_all_buckets(monkeypatch): - fake_cache = _make_spend_counter_cache( - redis_get_value=None, redis_increment_value=5.0 - ) + fake_cache = _make_spend_counter_cache(redis_get_value=None, redis_increment_value=5.0) fake_user_cache = _make_user_api_key_cache(get_value=None) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) @@ -543,9 +498,7 @@ async def test_increment_spend_counters_increments_all_buckets(monkeypatch): async def _fake_coalesced(**kwargs): return None - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(side_effect=_fake_coalesced) - ) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", AsyncMock(side_effect=_fake_coalesced)) await ps.increment_spend_counters( token="hashed-tok", @@ -554,25 +507,36 @@ async def test_increment_spend_counters_increments_all_buckets(monkeypatch): response_cost=5.0, ) + pipeline = fake_cache.redis_cache.async_increment_pipeline + pipeline.assert_awaited_once() + increment_list = pipeline.await_args.kwargs["increment_list"] + assert {op["key"] for op in increment_list} == { + "spend:key:hashed-tok", + "spend:team:t1", + "spend:team_member:u1:t1", + "spend:user:u1", + } + assert all(op["increment_value"] == 5.0 for op in increment_list) observed = { "redis_increment_called": fake_cache.redis_cache.async_increment.called, - "increment_calls": fake_cache.redis_cache.async_increment.call_count, + "pipeline_calls": pipeline.await_count, "user_cache_used": fake_user_cache.async_get_cache.called, } assert normalize(observed) == { - "redis_increment_called": True, - "increment_calls": 4, + "redis_increment_called": False, + "pipeline_calls": 1, "user_cache_used": True, } class _ConcurrencyProbe: - """Stand-in for redis_cache.async_increment that pins concurrency. + """Stand-in for redis_cache.async_get_cache that pins concurrency. - Each call registers itself as in-flight and blocks on ``release`` until the - test lets it proceed. ``all_arrived`` fires once ``expected`` distinct scope - increments are simultaneously suspended here, which can only happen if the - per-scope increments are gathered rather than awaited one after another. + Each warm-check read registers itself as in-flight and blocks on ``release`` + until the test lets it proceed. ``all_arrived`` fires once ``expected`` + distinct scope warm-checks are simultaneously suspended here, which can only + happen if the per-scope prepares are gathered rather than awaited one after + another. """ def __init__(self, expected_concurrency: int): @@ -581,36 +545,45 @@ class _ConcurrencyProbe: self.max_in_flight = 0 self.all_arrived = asyncio.Event() self.release = asyncio.Event() - self.values: dict[str, float] = {} + self.keys: list[str] = [] - async def async_increment(self, *, key, value, refresh_ttl=True): + async def async_get_cache(self, *, key, **kwargs): self.in_flight += 1 self.max_in_flight = max(self.max_in_flight, self.in_flight) + self.keys.append(key) if self.in_flight >= self.expected: self.all_arrived.set() if not self.release.is_set(): await self.release.wait() self.in_flight -= 1 - self.values[key] = self.values.get(key, 0.0) + value - return self.values[key] + return 1.0 @pytest.mark.asyncio async def test_increment_spend_counters_runs_scopes_concurrently(monkeypatch): """The six independent scopes (key, team, team_member, user, end_user+tags, - org) must be incremented concurrently. The probe only fires once all six are - suspended in async_increment at the same time, which is impossible if the + org) must prepare their increments concurrently. The probe only fires once + all eight warm-check reads (one per counter: 6 scopes + 2 tags) are + suspended in async_get_cache at the same time, which is impossible if the awaits are chained sequentially.""" - probe = _ConcurrencyProbe(expected_concurrency=6) - fake_cache = _make_spend_counter_cache(redis_get_value=None) - fake_cache.redis_cache.async_increment = probe.async_increment + probe = _ConcurrencyProbe(expected_concurrency=8) + fake_cache = _make_spend_counter_cache() + fake_cache.redis_cache.async_get_cache = probe.async_get_cache + recorded: dict[str, float] = {} + + async def _record_pipeline(increment_list, **_): + results = [] + for op in increment_list: + recorded[op["key"]] = recorded.get(op["key"], 0.0) + op["increment_value"] + results.append(recorded[op["key"]]) + return results + + fake_cache.redis_cache.async_increment_pipeline = AsyncMock(side_effect=_record_pipeline) fake_user_cache = _make_user_api_key_cache(get_value=None) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) monkeypatch.setattr(ps, "prisma_client", None) - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) - ) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)) task = asyncio.create_task( ps.increment_spend_counters( @@ -630,16 +603,16 @@ async def test_increment_spend_counters_runs_scopes_concurrently(monkeypatch): probe.release.set() await task pytest.fail( - "scope increments did not run concurrently; sequential awaits " - f"detected (peak in-flight was {probe.max_in_flight}, expected 6)" + "scope prepares did not run concurrently; sequential awaits " + f"detected (peak in-flight was {probe.max_in_flight}, expected 8)" ) - assert probe.in_flight == 6 - assert probe.max_in_flight == 6 + assert probe.in_flight == 8 + assert probe.max_in_flight == 8 probe.release.set() await task - assert probe.values == { + assert recorded == { "spend:key:hashed-tok": 5.0, "spend:team:t1": 5.0, "spend:team_member:u1:t1": 5.0, @@ -659,26 +632,25 @@ async def test_increment_spend_counters_skips_reserved_counter_keys(monkeypatch) import litellm.proxy.spend_tracking.budget_reservation as br reserved = {"spend:key:hashed-tok", "spend:org:org1"} - monkeypatch.setattr( - br, "get_reserved_counter_keys", MagicMock(return_value=set(reserved)) - ) + monkeypatch.setattr(br, "get_reserved_counter_keys", MagicMock(return_value=set(reserved))) monkeypatch.setattr(br, "reconcile_budget_reservation", AsyncMock()) recorded: dict[str, float] = {} - async def _record_increment(*, key, value, refresh_ttl=True): - recorded[key] = recorded.get(key, 0.0) + value - return recorded[key] + async def _record_pipeline(increment_list, **_): + results = [] + for op in increment_list: + recorded[op["key"]] = recorded.get(op["key"], 0.0) + op["increment_value"] + results.append(recorded[op["key"]]) + return results fake_cache = _make_spend_counter_cache(redis_get_value=None) - fake_cache.redis_cache.async_increment = _record_increment + fake_cache.redis_cache.async_increment_pipeline = AsyncMock(side_effect=_record_pipeline) fake_user_cache = _make_user_api_key_cache(get_value=None) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) monkeypatch.setattr(ps, "prisma_client", None) - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) - ) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)) reservation = {"finalized": False} await ps.increment_spend_counters( @@ -708,27 +680,46 @@ async def test_increment_spend_counters_failing_scope_propagates_after_siblings_ ): """A failure in one scope must propagate to the caller (so it can invalidate reserved counters) while every other scope still settles rather than being - left as an orphaned background task, and the reservation is not finalized.""" - recorded: dict[str, float] = {} + left as an orphaned background task, and the reservation is not finalized. + The surviving scopes' increments are still applied in the single pipeline: + dropping them would under-count spend, the unsafe direction for budget + enforcement.""" + warmed_keys: list[str] = [] - async def _increment(*, key, value, refresh_ttl=True): + async def _warm_check(*, key, **kwargs): + warmed_keys.append(key) if key == "spend:team:t1": - raise RuntimeError("redis increment failed") - recorded[key] = recorded.get(key, 0.0) + value - return recorded[key] + raise RuntimeError("redis get failed") + return 1.0 - fake_cache = _make_spend_counter_cache(redis_get_value=None) - fake_cache.redis_cache.async_increment = _increment + async def _reseed_fails(*, counter_key, **kwargs): + if counter_key == "spend:team:t1": + raise RuntimeError("reseed failed") + + applied: dict[str, float] = {} + + async def _record_pipeline(increment_list, **_): + results = [] + for op in increment_list: + applied[op["key"]] = op["increment_value"] + results.append(op["increment_value"]) + return results + + fake_cache = _make_spend_counter_cache() + fake_cache.redis_cache.async_get_cache = AsyncMock(side_effect=_warm_check) + fake_cache.redis_cache.async_increment_pipeline = AsyncMock(side_effect=_record_pipeline) fake_user_cache = _make_user_api_key_cache(get_value=None) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) monkeypatch.setattr(ps, "prisma_client", None) monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) + ps.SpendCounterReseed, + "coalesced", + AsyncMock(side_effect=_reseed_fails), ) reservation = {"finalized": False} - with pytest.raises(RuntimeError, match="redis increment failed"): + with pytest.raises(RuntimeError, match="reseed failed"): await ps.increment_spend_counters( token="hashed-tok", team_id="t1", @@ -741,7 +732,19 @@ async def test_increment_spend_counters_failing_scope_propagates_after_siblings_ ) assert reservation["finalized"] is False - assert recorded == { + # every sibling scope settled (its warm-check ran) before the error propagated + assert set(warmed_keys) == { + "spend:key:hashed-tok", + "spend:team:t1", + "spend:team_member:u1:t1", + "spend:user:u1", + "spend:end_user:eu1", + "spend:tag:a", + "spend:org:org1", + } + # the surviving scopes' increments were still applied, in one pipeline call + fake_cache.redis_cache.async_increment_pipeline.assert_awaited_once() + assert applied == { "spend:key:hashed-tok": 5.0, "spend:team_member:u1:t1": 5.0, "spend:user:u1": 5.0, @@ -749,6 +752,7 @@ async def test_increment_spend_counters_failing_scope_propagates_after_siblings_ "spend:tag:a": 5.0, "spend:org:org1": 5.0, } + fake_cache.redis_cache.async_increment.assert_not_awaited() @pytest.mark.asyncio @@ -772,6 +776,108 @@ async def test_increment_spend_counters_zero_cost_is_noop_finalizes_reservation( assert reservation == {"finalized": True} assert fake_cache.redis_cache.async_increment.called is False + fake_cache.redis_cache.async_increment_pipeline.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_increment_spend_counters_pipelines_all_scopes_in_one_redis_call( + monkeypatch, +): + """Every scope's increment must go out in a single async_increment_pipeline + call, not one INCRBYFLOAT round-trip per scope.""" + counter_cache = ps.DualCache() + fake_redis = AsyncMock() + fake_redis.async_get_cache = AsyncMock(return_value=1.0) # counters warm + + async def _pipeline(increment_list, **_): + return [1.5] * len(increment_list) + + fake_redis.async_increment_pipeline = AsyncMock(side_effect=_pipeline) + fake_redis.async_increment = AsyncMock() + fake_redis.get_ttl = MagicMock(return_value=None) + counter_cache.redis_cache = fake_redis + monkeypatch.setattr(ps, "spend_counter_cache", counter_cache) + monkeypatch.setattr(ps, "user_api_key_cache", ps.DualCache()) + monkeypatch.setattr(ps, "prisma_client", None) + + await ps.increment_spend_counters( + token="hashed", + team_id="team-1", + user_id="user-1", + response_cost=0.5, + org_id="org-1", + end_user_id="eu-1", + tags=["tag-a", "tag-b"], + ) + + fake_redis.async_increment_pipeline.assert_awaited_once() + assert fake_redis.async_increment.await_count == 0 + increment_list = fake_redis.async_increment_pipeline.await_args.kwargs["increment_list"] + expected_keys = { + "spend:key:hashed", + "spend:team:team-1", + "spend:team_member:user-1:team-1", + "spend:user:user-1", + "spend:end_user:eu-1", + "spend:tag:tag-a", + "spend:tag:tag-b", + "spend:org:org-1", + } + assert {op["key"] for op in increment_list} == expected_keys + assert all(op["increment_value"] == 0.5 for op in increment_list) + for key in expected_keys: + assert counter_cache.in_memory_cache.get_cache(key=key) == 1.5 + + +@pytest.mark.asyncio +async def test_increment_spend_counters_pipeline_failure_invalidates_all_counters( + monkeypatch, +): + """A failing pipeline must invalidate every pending counter so the next + request reseeds from the DB (which already holds this request's cost) + instead of trusting a value the write may have partially applied.""" + from redis.exceptions import MaxConnectionsError + + counter_cache = ps.DualCache() + pending_keys = ( + "spend:key:hashed", + "spend:team:team-1", + "spend:team_member:user-1:team-1", + "spend:user:user-1", + "spend:end_user:eu-1", + "spend:tag:tag-a", + "spend:tag:tag-b", + "spend:org:org-1", + ) + for key in pending_keys: + counter_cache.in_memory_cache.set_cache(key=key, value=1.0) + fake_redis = AsyncMock() + fake_redis.async_get_cache = AsyncMock(return_value=1.0) # counters warm + fake_redis.async_increment_pipeline = AsyncMock(side_effect=MaxConnectionsError()) + fake_redis.async_increment = AsyncMock() + fake_redis.async_delete_cache = AsyncMock() + fake_redis.get_ttl = MagicMock(return_value=None) + counter_cache.redis_cache = fake_redis + monkeypatch.setattr(ps, "spend_counter_cache", counter_cache) + monkeypatch.setattr(ps, "user_api_key_cache", ps.DualCache()) + monkeypatch.setattr(ps, "prisma_client", None) + + with pytest.raises(MaxConnectionsError): + await ps.increment_spend_counters( + token="hashed", + team_id="team-1", + user_id="user-1", + response_cost=0.5, + org_id="org-1", + end_user_id="eu-1", + tags=["tag-a", "tag-b"], + ) + + assert fake_redis.async_increment.await_count == 0 + deleted_keys = {call.kwargs["key"] for call in fake_redis.async_delete_cache.await_args_list} + assert deleted_keys == set(pending_keys) + for key in pending_keys: + assert counter_cache.in_memory_cache.get_cache(key=key) is None # --------------------------------------------------------------------------- @@ -781,9 +887,7 @@ async def test_increment_spend_counters_zero_cost_is_noop_finalizes_reservation( @pytest.mark.asyncio async def test_reconcile_budget_reservation_for_counter_update_returns_empty_set_when_none(): - result = await ps._reconcile_budget_reservation_for_counter_update( - budget_reservation=None, response_cost=1.0 - ) + result = await ps._reconcile_budget_reservation_for_counter_update(budget_reservation=None, response_cost=1.0) assert result == set() @@ -818,179 +922,151 @@ async def test_reconcile_budget_reservation_for_counter_update_failure_invalidat # --------------------------------------------------------------------------- -# _increment_end_user_and_tag_spend_counters +# _prepare_end_user_and_tag_spend_increments # --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_increment_end_user_and_tag_spend_counters_increments_each_unique_tag( +async def test_prepare_end_user_and_tag_spend_increments_returns_each_unique_tag( monkeypatch, ): - fake_cache = _make_spend_counter_cache( - redis_get_value=None, redis_increment_value=3.0 - ) + fake_cache = _make_spend_counter_cache(redis_get_value=1.0) fake_user_cache = _make_user_api_key_cache() monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) monkeypatch.setattr(ps, "prisma_client", None) - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) - ) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)) - await ps._increment_end_user_and_tag_spend_counters( + pending = await ps._prepare_end_user_and_tag_spend_increments( end_user_id="eu1", tags=["a", "b", "a", "", None], response_cost=3.0, reserved_counter_keys=set(), ) - observed = { - "increment_calls": fake_cache.redis_cache.async_increment.call_count, - "in_memory_set_calls": fake_cache.in_memory_cache.set_cache.call_count, - "called": fake_cache.redis_cache.async_increment.called, - } - assert normalize(observed) == { - "increment_calls": 3, - "in_memory_set_calls": 3, - "called": True, + assert {item.counter_key for item in pending} == { + "spend:end_user:eu1", + "spend:tag:a", + "spend:tag:b", } + assert all(item.increment == 3.0 for item in pending) @pytest.mark.asyncio -async def test_increment_end_user_and_tag_spend_counters_no_end_user_no_tags_invalid_input_noop( +async def test_prepare_end_user_and_tag_spend_increments_no_end_user_no_tags_invalid_input_noop( monkeypatch, ): fake_cache = _make_spend_counter_cache() monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - await ps._increment_end_user_and_tag_spend_counters( + pending = await ps._prepare_end_user_and_tag_spend_increments( end_user_id=None, tags=None, response_cost=1.0, reserved_counter_keys=set(), ) + assert pending == () assert fake_cache.redis_cache.async_increment.called is False # --------------------------------------------------------------------------- -# _increment_org_spend_counter +# _prepare_org_spend_increment # --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_increment_org_spend_counter_increments_when_org_present(monkeypatch): - fake_cache = _make_spend_counter_cache( - redis_get_value=None, redis_increment_value=10.0 - ) +async def test_prepare_org_spend_increment_returns_pending_when_org_present(monkeypatch): + fake_cache = _make_spend_counter_cache(redis_get_value=1.0) fake_user_cache = _make_user_api_key_cache() monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) monkeypatch.setattr(ps, "prisma_client", None) - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) - ) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)) - await ps._increment_org_spend_counter( + pending = await ps._prepare_org_spend_increment( org_id="org-1", response_cost=10.0, reserved_counter_keys=set(), ) - observed = { - "increment_called": fake_cache.redis_cache.async_increment.called, - "increment_calls": fake_cache.redis_cache.async_increment.call_count, - "counter_key_arg": fake_cache.redis_cache.async_increment.call_args.kwargs[ - "key" - ], - } - assert normalize(observed) == { - "increment_called": True, - "increment_calls": 1, - "counter_key_arg": "spend:org:org-1", - } + assert len(pending) == 1 + assert pending[0].counter_key == "spend:org:org-1" + assert pending[0].increment == 10.0 @pytest.mark.asyncio -async def test_increment_org_spend_counter_no_org_is_noop_invalid_id(monkeypatch): +async def test_prepare_org_spend_increment_no_org_is_noop_invalid_id(monkeypatch): fake_cache = _make_spend_counter_cache() monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - await ps._increment_org_spend_counter( + pending = await ps._prepare_org_spend_increment( org_id=None, response_cost=1.0, reserved_counter_keys=set(), ) + assert pending == () assert fake_cache.redis_cache.async_increment.called is False # --------------------------------------------------------------------------- -# _init_and_increment_unreserved_spend_counter +# _prepare_unreserved_spend_counter_increment # --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_init_and_increment_unreserved_spend_counter_skips_reserved_keys( +async def test_prepare_unreserved_spend_counter_increment_skips_reserved_keys( monkeypatch, ): fake_cache = _make_spend_counter_cache() monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - await ps._init_and_increment_unreserved_spend_counter( + pending = await ps._prepare_unreserved_spend_counter_increment( counter_key="spend:tag:x", source_cache_key="tag:x", increment=1.0, reserved_counter_keys={"spend:tag:x"}, ) + assert pending is None assert fake_cache.redis_cache.async_increment.called is False @pytest.mark.asyncio -async def test_init_and_increment_unreserved_spend_counter_proceeds_when_not_reserved( +async def test_prepare_unreserved_spend_counter_increment_proceeds_when_not_reserved( monkeypatch, ): - fake_cache = _make_spend_counter_cache( - redis_get_value=None, redis_increment_value=2.0 - ) + fake_cache = _make_spend_counter_cache(redis_get_value=None) fake_user_cache = _make_user_api_key_cache() + reseed = AsyncMock(return_value=None) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) monkeypatch.setattr(ps, "prisma_client", None) - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) - ) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", reseed) - await ps._init_and_increment_unreserved_spend_counter( + pending = await ps._prepare_unreserved_spend_counter_increment( counter_key="spend:tag:y", source_cache_key="tag:y", increment=2.0, reserved_counter_keys=set(), ) - observed = { - "increment_called": fake_cache.redis_cache.async_increment.called, - "redis_get_called": fake_cache.redis_cache.async_get_cache.called, - "reseed_consulted": True, - } - assert observed == { - "increment_called": True, - "redis_get_called": True, - "reseed_consulted": True, - } + assert pending is not None + assert pending.counter_key == "spend:tag:y" + assert pending.increment == 2.0 + assert fake_cache.redis_cache.async_get_cache.called is True + assert reseed.called is True # --------------------------------------------------------------------------- -# _init_and_increment_spend_counter +# _prepare_spend_counter_increment # --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_init_and_increment_spend_counter_warm_cache_skips_reseed(monkeypatch): - fake_cache = _make_spend_counter_cache( - redis_get_value=11.0, redis_increment_value=14.0 - ) +async def test_prepare_spend_counter_increment_warm_cache_skips_reseed(monkeypatch): + fake_cache = _make_spend_counter_cache(redis_get_value=11.0) fake_user_cache = _make_user_api_key_cache() monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) @@ -998,12 +1074,14 @@ async def test_init_and_increment_spend_counter_warm_cache_skips_reseed(monkeypa reseed = AsyncMock(return_value=None) monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", reseed) - await ps._init_and_increment_spend_counter( + pending = await ps._prepare_spend_counter_increment( counter_key="spend:key:k", source_cache_key="k", increment=3.0, ) + assert pending.counter_key == "spend:key:k" + assert pending.increment == 3.0 observed = { "reseed_called": reseed.called, "increment_called": fake_cache.redis_cache.async_increment.called, @@ -1011,23 +1089,21 @@ async def test_init_and_increment_spend_counter_warm_cache_skips_reseed(monkeypa } assert normalize(observed) == { "reseed_called": False, - "increment_called": True, + "increment_called": False, "in_memory_seeded_from_redis": True, } # --------------------------------------------------------------------------- -# _init_and_increment_window_spend_counter +# _prepare_window_spend_counter_increment # --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_init_and_increment_window_spend_counter_increments_when_initialized( +async def test_prepare_window_spend_counter_increment_returns_pending_when_initialized( monkeypatch, ): - fake_cache = _make_spend_counter_cache( - redis_get_value=0.0, redis_increment_value=5.0 - ) + fake_cache = _make_spend_counter_cache(redis_get_value=0.0) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "prisma_client", None) monkeypatch.setattr( @@ -1036,7 +1112,7 @@ async def test_init_and_increment_window_spend_counter_increments_when_initializ AsyncMock(return_value=0.0), ) - await ps._init_and_increment_window_spend_counter( + pending = await ps._prepare_window_spend_counter_increment( counter_key="spend:key:k:window:1d", entity_type="Key", entity_id="k", @@ -1045,26 +1121,19 @@ async def test_init_and_increment_window_spend_counter_increments_when_initializ increment=5.0, ) - observed = { - "redis_increment_called": fake_cache.redis_cache.async_increment.called, - "increment_calls": fake_cache.redis_cache.async_increment.call_count, - "in_memory_set_calls": fake_cache.in_memory_cache.set_cache.call_count, - } - assert normalize(observed) == { - "redis_increment_called": True, - "increment_calls": 1, - "in_memory_set_calls": 2, - } + assert pending is not None + assert pending.counter_key == "spend:key:k:window:1d" + assert pending.increment == 5.0 @pytest.mark.asyncio -async def test_init_and_increment_window_spend_counter_missing_window_start_invalid_skips( +async def test_prepare_window_spend_counter_increment_missing_window_start_invalid_skips( monkeypatch, ): fake_cache = _make_spend_counter_cache() monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - await ps._init_and_increment_window_spend_counter( + pending = await ps._prepare_window_spend_counter_increment( counter_key="spend:key:k:window:1d", entity_type="Key", entity_id="k", @@ -1073,6 +1142,7 @@ async def test_init_and_increment_window_spend_counter_missing_window_start_inva increment=5.0, ) + assert pending is None assert fake_cache.redis_cache.async_increment.called is False @@ -1114,16 +1184,12 @@ async def test_ensure_spend_counter_initialized_warm_skips_reseed_and_source( async def test_ensure_spend_counter_initialized_cold_seeds_from_source_cache( monkeypatch, ): - fake_cache = _make_spend_counter_cache( - redis_get_value=None, redis_increment_value=7.0 - ) + fake_cache = _make_spend_counter_cache(redis_get_value=None, redis_increment_value=7.0) fake_user_cache = _make_user_api_key_cache(get_value={"spend": 7.0}) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) monkeypatch.setattr(ps, "prisma_client", None) - monkeypatch.setattr( - ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None) - ) + monkeypatch.setattr(ps.SpendCounterReseed, "coalesced", AsyncMock(return_value=None)) await ps._ensure_spend_counter_initialized( counter_key="spend:user:u", @@ -1163,9 +1229,7 @@ async def test_get_source_cache_base_spend_reads_first_hit_from_list(monkeypatch fake_user_cache.async_get_cache = AsyncMock(side_effect=_get) monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache) - result = await ps._get_source_cache_base_spend( - source_cache_key=["miss", "hit-obj", "miss2"] - ) + result = await ps._get_source_cache_base_spend(source_cache_key=["miss", "hit-obj", "miss2"]) observed = { "result": result, @@ -1294,9 +1358,7 @@ async def test_increment_spend_counter_cache_redis_path_returns_new_value(monkey fake_cache = _make_spend_counter_cache(redis_increment_value=44.0) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - result = await ps._increment_spend_counter_cache( - counter_key="spend:key:k", increment=4.0 - ) + result = await ps._increment_spend_counter_cache(counter_key="spend:key:k", increment=4.0) observed = { "result": result, @@ -1314,15 +1376,11 @@ async def test_increment_spend_counter_cache_redis_path_returns_new_value(monkey async def test_increment_spend_counter_cache_redis_error_raises_and_invalidates( monkeypatch, ): - fake_cache = _make_spend_counter_cache( - redis_increment_side_effect=RuntimeError("incr fail") - ) + fake_cache = _make_spend_counter_cache(redis_increment_side_effect=RuntimeError("incr fail")) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) with pytest.raises(RuntimeError): - await ps._increment_spend_counter_cache( - counter_key="spend:key:k", increment=1.0 - ) + await ps._increment_spend_counter_cache(counter_key="spend:key:k", increment=1.0) assert fake_cache.in_memory_cache.delete_cache.called is True assert fake_cache.redis_cache.async_delete_cache.called is True @@ -1343,9 +1401,7 @@ async def test_invalidate_spend_counter_deletes_in_memory_and_redis(monkeypatch) observed = { "in_memory_delete_called": fake_cache.in_memory_cache.delete_cache.called, "redis_delete_called": fake_cache.redis_cache.async_delete_cache.called, - "delete_args_key": fake_cache.redis_cache.async_delete_cache.call_args.kwargs[ - "key" - ], + "delete_args_key": fake_cache.redis_cache.async_delete_cache.call_args.kwargs["key"], } assert normalize(observed) == { "in_memory_delete_called": True, @@ -1357,9 +1413,7 @@ async def test_invalidate_spend_counter_deletes_in_memory_and_redis(monkeypatch) @pytest.mark.asyncio async def test_invalidate_spend_counter_swallows_redis_failure_no_raise(monkeypatch): fake_cache = _make_spend_counter_cache() - fake_cache.redis_cache.async_delete_cache = AsyncMock( - side_effect=RuntimeError("redis down") - ) + fake_cache.redis_cache.async_delete_cache = AsyncMock(side_effect=RuntimeError("redis down")) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) await ps._invalidate_spend_counter(counter_key="spend:key:k") diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation_redis_failure.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation_redis_failure.py index c123eeeed36..6165af4920d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation_redis_failure.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation_redis_failure.py @@ -41,6 +41,15 @@ class _FlakyRedisCache: self._store[key] = float(value) return True + async def async_increment_pipeline(self, increment_list, **kwargs): + results = [] + for op in increment_list: + results.append(await self.async_increment(op["key"], op["increment_value"])) + return results + + def get_ttl(self, **kwargs): + return None + @pytest.mark.asyncio async def test_direct_increment_runs_when_reservation_reconcile_hits_redis_failure( diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 6dab054d8ea..40ebc03781c 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -2220,6 +2220,15 @@ class _ExpiringRedisCache: async def async_delete_cache(self, key: str, *args: object, **kwargs: object) -> None: self.store.pop(key, None) + async def async_increment_pipeline(self, increment_list, **kwargs): + results = [] + for op in increment_list: + results.append(await self.async_increment(op["key"], op["increment_value"])) + return results + + def get_ttl(self, **kwargs) -> None: + return None + @pytest.mark.asyncio async def test_reconcile_after_redis_counter_expiry_keeps_request_cost_enforced( diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 54579e6cb7c..b0f38978727 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -7749,7 +7749,7 @@ async def test_increment_spend_counters_team_and_member(): @pytest.mark.asyncio -async def test_init_and_increment_spend_counter_reseeds_from_db_on_counter_miss(): +async def test_prepare_spend_counter_increment_reseeds_from_db_on_counter_miss(): """When the Redis counter is missing, the reseed path reads the authoritative spend from the DB (not a stale cache), so the next increment continues from the correct base value.""" @@ -7762,8 +7762,17 @@ async def test_init_and_increment_spend_counter_reseeds_from_db_on_counter_miss( recorded_increments.append({"key": key, "value": value, "ttl": ttl}) return value + async def record_pipeline(increment_list, **kwargs): + results = [] + for op in increment_list: + await record_increment(key=op["key"], value=op["increment_value"], ttl=op["ttl"]) + results.append(op["increment_value"]) + return results + fake_redis = AsyncMock() fake_redis.async_increment = AsyncMock(side_effect=record_increment) + fake_redis.async_increment_pipeline = AsyncMock(side_effect=record_pipeline) + fake_redis.get_ttl = MagicMock(return_value=None) fake_redis.async_get_cache = AsyncMock(return_value=None) # counter missing fake_redis.async_set_cache = AsyncMock(return_value=True) # SET NX wins counter_cache.redis_cache = fake_redis @@ -7782,7 +7791,10 @@ async def test_init_and_increment_spend_counter_reseeds_from_db_on_counter_miss( stale_cache.in_memory_cache.set_cache(key="team_id:team-9", value=stale_team) import litellm.proxy.proxy_server as ps - from litellm.proxy.proxy_server import _init_and_increment_spend_counter + from litellm.proxy.proxy_server import ( + _apply_spend_counter_increments, + _prepare_spend_counter_increment, + ) orig_user, orig_counter, orig_prisma = ( ps.user_api_key_cache, @@ -7793,11 +7805,12 @@ async def test_init_and_increment_spend_counter_reseeds_from_db_on_counter_miss( ps.spend_counter_cache = counter_cache ps.prisma_client = fake_prisma try: - await _init_and_increment_spend_counter( + pending = await _prepare_spend_counter_increment( counter_key="spend:team:team-9", source_cache_key="team_id:team-9", increment=1.5, ) + await _apply_spend_counter_increments(pending=(pending,)) fake_prisma.db.litellm_teamtable.find_unique.assert_awaited_once_with(where={"team_id": "team-9"}) # Seed uses SET NX with db_spend (42) — cross-pod safe, no INCR of 42. @@ -7976,7 +7989,10 @@ async def test_reseed_spend_from_db_skips_window_variant_keys(): @pytest.mark.asyncio async def test_window_spend_counter_reseeds_from_spend_logs_on_counter_miss(): from litellm.caching.dual_cache import DualCache - from litellm.proxy.proxy_server import _init_and_increment_window_spend_counter + from litellm.proxy.proxy_server import ( + _apply_spend_counter_increments, + _prepare_window_spend_counter_increment, + ) counter_cache = DualCache() window_start = datetime.now(timezone.utc) - timedelta(hours=1) @@ -7992,7 +8008,7 @@ async def test_window_spend_counter_reseeds_from_spend_logs_on_counter_miss(): ps.spend_counter_cache = counter_cache ps.prisma_client = fake_prisma try: - await _init_and_increment_window_spend_counter( + pending = await _prepare_window_spend_counter_increment( counter_key="spend:key:key-window:window:1h", entity_type="Key", entity_id="key-window", @@ -8000,6 +8016,7 @@ async def test_window_spend_counter_reseeds_from_spend_logs_on_counter_miss(): window_start=window_start, increment=0.5, ) + await _apply_spend_counter_increments(pending=(pending,) if pending is not None else ()) fake_prisma.db.litellm_spendlogs.group_by.assert_awaited_once_with( by=["api_key"], @@ -8015,7 +8032,10 @@ async def test_window_spend_counter_reseeds_from_spend_logs_on_counter_miss(): @pytest.mark.asyncio async def test_init_spend_counter_redis_clean_miss_skips_stale_in_memory(): from litellm.caching.dual_cache import DualCache - from litellm.proxy.proxy_server import _init_and_increment_spend_counter + from litellm.proxy.proxy_server import ( + _apply_spend_counter_increments, + _prepare_spend_counter_increment, + ) counter_cache = DualCache() counter_key = "spend:team:team-stale-local" @@ -8037,6 +8057,15 @@ async def test_init_spend_counter_redis_clean_miss_skips_stale_in_memory(): fake_redis.async_get_cache = AsyncMock(return_value=None) fake_redis.async_increment = AsyncMock(side_effect=redis_increment) fake_redis.async_set_cache = AsyncMock(side_effect=redis_set_cache) + + async def redis_increment_pipeline(increment_list, **_): + results = [] + for op in increment_list: + results.append(await redis_increment(key=op["key"], value=op["increment_value"])) + return results + + fake_redis.async_increment_pipeline = AsyncMock(side_effect=redis_increment_pipeline) + fake_redis.get_ttl = MagicMock(return_value=None) counter_cache.redis_cache = fake_redis db_row = MagicMock() @@ -8055,11 +8084,12 @@ async def test_init_spend_counter_redis_clean_miss_skips_stale_in_memory(): ps.prisma_client = fake_prisma ps.user_api_key_cache = DualCache() try: - await _init_and_increment_spend_counter( + pending = await _prepare_spend_counter_increment( counter_key=counter_key, source_cache_key="team_id:team-stale-local", increment=1.5, ) + await _apply_spend_counter_increments(pending=(pending,)) fake_prisma.db.litellm_teamtable.find_unique.assert_awaited_once_with(where={"team_id": "team-stale-local"}) # Seed via SET NX (42) + delta via INCRBYFLOAT (1.5) = 43.5. @@ -8074,7 +8104,10 @@ async def test_init_spend_counter_redis_clean_miss_skips_stale_in_memory(): @pytest.mark.asyncio async def test_window_spend_counter_redis_clean_miss_skips_stale_in_memory(): from litellm.caching.dual_cache import DualCache - from litellm.proxy.proxy_server import _init_and_increment_window_spend_counter + from litellm.proxy.proxy_server import ( + _apply_spend_counter_increments, + _prepare_window_spend_counter_increment, + ) counter_cache = DualCache() counter_key = "spend:key:key-window-stale-local:window:1h" @@ -8097,6 +8130,15 @@ async def test_window_spend_counter_redis_clean_miss_skips_stale_in_memory(): fake_redis.async_get_cache = AsyncMock(return_value=None) fake_redis.async_set_cache = AsyncMock(side_effect=redis_set_cache) fake_redis.async_increment = AsyncMock(side_effect=redis_increment) + + async def redis_increment_pipeline(increment_list, **_): + results = [] + for op in increment_list: + results.append(await redis_increment(key=op["key"], value=op["increment_value"])) + return results + + fake_redis.async_increment_pipeline = AsyncMock(side_effect=redis_increment_pipeline) + fake_redis.get_ttl = MagicMock(return_value=None) counter_cache.redis_cache = fake_redis fake_prisma = MagicMock() @@ -8111,7 +8153,7 @@ async def test_window_spend_counter_redis_clean_miss_skips_stale_in_memory(): ps.spend_counter_cache = counter_cache ps.prisma_client = fake_prisma try: - await _init_and_increment_window_spend_counter( + pending = await _prepare_window_spend_counter_increment( counter_key=counter_key, entity_type="Key", entity_id="key-window-stale-local", @@ -8119,6 +8161,7 @@ async def test_window_spend_counter_redis_clean_miss_skips_stale_in_memory(): window_start=window_start, increment=0.5, ) + await _apply_spend_counter_increments(pending=(pending,) if pending is not None else ()) fake_prisma.db.litellm_spendlogs.group_by.assert_awaited_once_with( by=["api_key"], @@ -8138,7 +8181,10 @@ async def test_window_spend_counter_redis_clean_miss_skips_stale_in_memory(): @pytest.mark.asyncio async def test_window_spend_counter_redis_concurrent_seed_does_not_double_seed(): from litellm.caching.dual_cache import DualCache - from litellm.proxy.proxy_server import _init_and_increment_window_spend_counter + from litellm.proxy.proxy_server import ( + _apply_spend_counter_increments, + _prepare_window_spend_counter_increment, + ) counter_cache = DualCache() counter_key = "spend:key:key-window-concurrent-seed:window:1h" @@ -8161,6 +8207,15 @@ async def test_window_spend_counter_redis_concurrent_seed_does_not_double_seed() fake_redis.async_get_cache = AsyncMock(side_effect=redis_get_cache) fake_redis.async_set_cache = AsyncMock(return_value=False) fake_redis.async_increment = AsyncMock(side_effect=redis_increment) + + async def redis_increment_pipeline(increment_list, **_): + results = [] + for op in increment_list: + results.append(await redis_increment(key=op["key"], value=op["increment_value"])) + return results + + fake_redis.async_increment_pipeline = AsyncMock(side_effect=redis_increment_pipeline) + fake_redis.get_ttl = MagicMock(return_value=None) counter_cache.redis_cache = fake_redis fake_prisma = MagicMock() @@ -8175,7 +8230,7 @@ async def test_window_spend_counter_redis_concurrent_seed_does_not_double_seed() ps.spend_counter_cache = counter_cache ps.prisma_client = fake_prisma try: - await _init_and_increment_window_spend_counter( + pending = await _prepare_window_spend_counter_increment( counter_key=counter_key, entity_type="Key", entity_id="key-window-concurrent-seed", @@ -8183,6 +8238,7 @@ async def test_window_spend_counter_redis_concurrent_seed_does_not_double_seed() window_start=window_start, increment=0.5, ) + await _apply_spend_counter_increments(pending=(pending,) if pending is not None else ()) fake_redis.async_set_cache.assert_awaited_once_with( key=counter_key, @@ -8199,7 +8255,7 @@ async def test_window_spend_counter_redis_concurrent_seed_does_not_double_seed() @pytest.mark.asyncio async def test_window_spend_counter_skips_invalid_window_start(): from litellm.caching.dual_cache import DualCache - from litellm.proxy.proxy_server import _init_and_increment_window_spend_counter + from litellm.proxy.proxy_server import _prepare_window_spend_counter_increment counter_cache = DualCache() @@ -8208,7 +8264,7 @@ async def test_window_spend_counter_skips_invalid_window_start(): orig_counter = ps.spend_counter_cache ps.spend_counter_cache = counter_cache try: - await _init_and_increment_window_spend_counter( + pending = await _prepare_window_spend_counter_increment( counter_key="spend:key:key-invalid-window:window:not-a-duration", entity_type="Key", entity_id="key-invalid-window", @@ -8216,6 +8272,7 @@ async def test_window_spend_counter_skips_invalid_window_start(): window_start=None, increment=0.5, ) + assert pending is None assert counter_cache.in_memory_cache.get_cache(key="spend:key:key-invalid-window:window:not-a-duration") is None finally: @@ -8279,6 +8336,9 @@ async def test_increment_spend_counters_finalizes_after_unreserved_increments(): async def assert_reservation_not_finalized_yet(**kwargs): assert budget_reservation["finalized"] is False incremented_counters.append(kwargs["counter_key"]) + return ps._PendingSpendIncrement( + counter_key=kwargs["counter_key"], increment=kwargs["increment"] + ) import litellm.proxy.proxy_server as ps @@ -8287,7 +8347,7 @@ async def test_increment_spend_counters_finalizes_after_unreserved_increments(): ps.user_api_key_cache = DualCache() try: with patch( - "litellm.proxy.proxy_server._init_and_increment_spend_counter", + "litellm.proxy.proxy_server._prepare_spend_counter_increment", new=AsyncMock(side_effect=assert_reservation_not_finalized_yet), ): await increment_spend_counters( @@ -8620,7 +8680,7 @@ async def test_get_current_spend_uses_db_zero_over_stale_fallback(): async def test_concurrent_read_and_write_paths_share_one_db_query(): """ The read path (`get_current_spend`) and the write path - (`_init_and_increment_spend_counter`) both reseed cold counters from + (`_prepare_spend_counter_increment`) both reseed cold counters from the DB. They must share the per-counter lock so a concurrent pre-call enforcement read and post-call increment for the same counter collapse to one DB query, not two. @@ -8629,7 +8689,7 @@ async def test_concurrent_read_and_write_paths_share_one_db_query(): from litellm.caching.dual_cache import DualCache from litellm.proxy.proxy_server import ( - _init_and_increment_spend_counter, + _prepare_spend_counter_increment, get_current_spend, ) @@ -8683,7 +8743,7 @@ async def test_concurrent_read_and_write_paths_share_one_db_query(): try: results = await _asyncio.gather( get_current_spend(counter_key=counter_key, fallback_spend=0.0), - _init_and_increment_spend_counter( + _prepare_spend_counter_increment( counter_key=counter_key, source_cache_key="ignored", increment=1.5, From 7c6e33ef70889445fab5650b1490ddf3c0448a11 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:38:26 -0700 Subject: [PATCH 196/411] fix(proxy): ignore team_id="" on /key/update so team-less keys can be updated and imported (#40421) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 8 ++++++ .../test_key_management_endpoints.py | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index df7ee8f508c..0e2fd50ac9f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3,6 +3,7 @@ import json import os from collections.abc import Callable, Mapping from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple import httpx @@ -1294,6 +1295,13 @@ class UpdateKeyRequest(KeyRequestBase): rotation_interval: str | None = None organization_id: str | None = None + @model_validator(mode="before") + @classmethod + def drop_blank_team_id(cls, values: object) -> object: + if isinstance(values, Mapping) and values.get("team_id") == "": + return MappingProxyType({k: v for k, v in values.items() if k != "team_id"}) + return values + @field_validator("organization_id", mode="before") @classmethod def treat_cleared_organization_id_as_unset(cls, v: object) -> object: diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index a873a367eab..d2aeba18f7d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -17975,6 +17975,32 @@ def test_key_request_blank_organization_id_is_unset(): assert UpdateKeyRequest(key="sk-1", organization_id="org-1").organization_id == "org-1" +def test_update_key_request_blank_team_id_is_not_a_team_change(): + from litellm.proxy._types import UpdateKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + is_different_team, + ) + + blank = UpdateKeyRequest(key="sk-1", team_id="", key_alias="renamed") + assert blank.team_id is None + assert "team_id" not in blank.model_dump(exclude_unset=True) + assert blank.model_dump(exclude_unset=True) == {"key": "sk-1", "key_alias": "renamed"} + assert is_different_team(data=blank, existing_key_row=LiteLLM_VerificationToken(token="hashed")) is False + assert ( + is_different_team(data=blank, existing_key_row=LiteLLM_VerificationToken(token="hashed", team_id="team-1")) + is False + ) + assert "team_id" in UpdateKeyRequest(key="sk-1", team_id=None).model_dump(exclude_unset=True) + assert UpdateKeyRequest(key="sk-1", team_id="team-1").team_id == "team-1" + assert ( + is_different_team( + data=UpdateKeyRequest(key="sk-1", team_id="team-1"), + existing_key_row=LiteLLM_VerificationToken(token="hashed"), + ) + is True + ) + + def test_key_generation_check_blank_team_id_uses_personal_permissions(monkeypatch): """key_generation_check with team_id="" must take the personal-key path instead of failing the team lookup with "Unable to find team object" (LIT-3925).""" From 699ae63b2a3270ab8b1068f8beba4a3bf71543b9 Mon Sep 17 00:00:00 2001 From: Clement Date: Thu, 10 Sep 2026 01:47:34 +0800 Subject: [PATCH 197/411] feat(router): support percentile-based TTFT routing (#40352) * feat(router): support percentile-based TTFT routing * fix(router): apply routing_strategy_args updates to the live selector Runtime routing_strategy_args updates (config reload, update_settings) only rebuilt the strategy selector when routing_strategy itself changed, so a newly added ttft_percentile sat unused until the proxy restarted. Also drops a comment that only restated the code it sat above. Claude-Session: https://claude.ai/code/session_01PmqjhFYcUh6vA72d8W9gdB * refactor(router): drop unreachable empty-samples guard in percentile latency _percentile_latency is only called behind use_ttft, which already requires a non-empty ttft sample list, so the early return was dead code and the one line Codecov flagged as uncovered on this patch. Claude-Session: https://claude.ai/code/session_01PmqjhFYcUh6vA72d8W9gdB * test(router): cover the no-selector path of a routing_strategy_args update simple-shuffle has no selector attribute to re-link, so the early return guards a setattr with a None attribute name. Dropping the guard makes the new test fail with "attribute name must be string, not 'NoneType'". Claude-Session: https://claude.ai/code/session_01PmqjhFYcUh6vA72d8W9gdB * fix(test): assert ValidationError on out-of-range ttft_percentile pytest.raises(ValueError) tripped PT011 for being too broad. Pydantic raises ValidationError for the gt/le constraint, so naming it satisfies the rule and pins the assertion to the constraint under test. Claude-Session: https://claude.ai/code/session_01PmqjhFYcUh6vA72d8W9gdB * fix(router): drop Final from a per-deployment loop variable basedpyright rejects "A Final variable cannot be assigned within a loop", which pushed reportGeneralTypeIssues one over its budget. selected_latency is rebound each iteration, so it matches its unannotated neighbours in the same loop. Claude-Session: https://claude.ai/code/session_01PmqjhFYcUh6vA72d8W9gdB * test(router): exempt _apply_updated_routing_strategy_args from the name scan The scan only reads test files with "router" in the filename, so it cannot see the update_settings tests in router_strategy/test_lowest_latency.py. Calling the private helper directly would test structure rather than behaviour, so it joins the existing entries ignored for the same reason. Claude-Session: https://claude.ai/code/session_01PmqjhFYcUh6vA72d8W9gdB --- litellm/router.py | 47 ++++- litellm/router_strategy/lowest_latency.py | 21 ++- .../router_code_coverage.py | 1 + .../router_strategy/test_lowest_latency.py | 174 +++++++++++++++++- 4 files changed, 225 insertions(+), 18 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 934a4ac86a9..2fbdf541487 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1317,6 +1317,43 @@ class Router: if isinstance(litellm.input_callback, list): litellm.input_callback = [c for c in litellm.input_callback if id(c) not in selector_ids] + def _apply_updated_routing_strategy_args(self) -> None: + """ + Re-link the default group's selector to the current `routing_strategy_args`. + + Selectors freeze their `RoutingArgs` at construction, so a runtime args + update would otherwise keep serving the boot-time values until restart. + Latency/usage state survives the rebuild: it lives in the shared router + cache, not on the selector. + """ + strategy: Final = self._normalize_strategy(self.routing_strategy) + if strategy == "lar1": + from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy + + apply_lar1_routing_strategy(self, self.routing_strategy_args) + return + + attr: Final = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy or "") + current: Final = getattr(self, attr, None) if attr is not None else None + if attr is None or current is None: + return + + try: + rebuilt: Final = self._build_strategy_selector( + strategy=strategy or "", + routing_strategy_args=self.routing_strategy_args, + ) + except (TypeError, ValidationError): + verbose_router_logger.exception( + "Invalid routing_strategy_args %s for '%s'; keeping the previous ones", + self.routing_strategy_args, + strategy, + ) + return + + self._unregister_router_selectors((current,)) + setattr(self, attr, rebuilt) + def routing_strategy_init(self, routing_strategy: RoutingStrategy | str, routing_strategy_args: dict): verbose_router_logger.info("Routing strategy: %s", routing_strategy) self._validate_routing_strategy(routing_strategy) @@ -11847,7 +11884,7 @@ class Router: _existing_router_settings: Final = self.get_settings() rebuild_routing_groups = False - relink_lar1_from_args = False + routing_args_updated = False for var in kwargs: if var in RUNTIME_UPDATABLE_ROUTER_SETTINGS: if var in _int_settings: @@ -11886,15 +11923,13 @@ class Router: ) rebuild_routing_groups = True elif var == "routing_strategy_args": - relink_lar1_from_args = True + routing_args_updated = True setattr(self, var, value) else: verbose_router_logger.debug("Setting %s is not allowed", var) - if relink_lar1_from_args and self._normalize_strategy(self.routing_strategy) == "lar1": - from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy - - apply_lar1_routing_strategy(self, self.routing_strategy_args) + if routing_args_updated: + self._apply_updated_routing_strategy_args() if rebuild_routing_groups: self._init_routing_groups(self._routing_groups_input) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 805d4ff9080..e902192811c 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -3,8 +3,11 @@ import random from collections.abc import Sequence from datetime import datetime, timedelta +from math import ceil from typing import TYPE_CHECKING, Any, Final +from pydantic import Field + import litellm from litellm import ModelResponse, token_counter, verbose_logger from litellm.caching.caching import DualCache @@ -24,6 +27,7 @@ class RoutingArgs(LiteLLMPydanticObjectBase): ttl: float = 1 * 60 * 60 # 1 hour lowest_latency_buffer: float = 0 max_latency_list_size: int = 10 + ttft_percentile: float | None = Field(default=None, gt=0, le=1) def _average_latency(samples: Sequence[float]) -> float: @@ -32,6 +36,12 @@ def _average_latency(samples: Sequence[float]) -> float: return sum(samples) / len(samples) +def _percentile_latency(samples: Sequence[float], percentile: float) -> float: + values: Final = sorted(samples) + index: Final = ceil(len(values) * percentile) - 1 + return values[index] + + def _ttft_seconds(elapsed: timedelta | float) -> float: if isinstance(elapsed, timedelta): return elapsed.total_seconds() @@ -427,14 +437,17 @@ class LowestLatencyLoggingHandler(CustomLogger): item_rpm = item_map.get(precise_minute, {}).get("rpm", 0) item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) - # get average latency or average ttft (depending on streaming/non-streaming) use_ttft = ( request_kwargs is not None and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True and len(item_ttft_latency) > 0 ) - average_latency = _average_latency(item_ttft_latency if use_ttft else item_latency) + selected_latency = ( + _percentile_latency(item_ttft_latency, self.routing_args.ttft_percentile) + if use_ttft and self.routing_args.ttft_percentile is not None + else _average_latency(item_ttft_latency if use_ttft else item_latency) + ) # -------------- # # Debugging Logic @@ -443,7 +456,7 @@ class LowestLatencyLoggingHandler(CustomLogger): # this helps a user to debug why the router picked a specfic deployment # _deployment_api_base = _deployment.get("litellm_params", {}).get("api_base", "") if _deployment_api_base is not None: - _latency_per_deployment[_deployment_api_base] = average_latency + _latency_per_deployment[_deployment_api_base] = selected_latency # -------------- # # End of Debugging Logic # -------------- # @@ -453,7 +466,7 @@ class LowestLatencyLoggingHandler(CustomLogger): ): # if user passed in tpm / rpm in the model_list continue else: - potential_deployments.append((_deployment, average_latency)) + potential_deployments.append((_deployment, selected_latency)) if len(potential_deployments) == 0: return None diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 0af29f069c6..a11f015743b 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -87,6 +87,7 @@ ignored_function_names = [ "_delete_claude_code_session_router_binding", # Tested through Redis cleanup failure in test_router.py "_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py "_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py + "_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name) ] diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py index 812d7bbff32..1a8614e3fca 100644 --- a/tests/test_litellm/router_strategy/test_lowest_latency.py +++ b/tests/test_litellm/router_strategy/test_lowest_latency.py @@ -8,11 +8,12 @@ import json from datetime import datetime, timedelta import pytest - +from pydantic import ValidationError import litellm from litellm.caching.caching import DualCache -from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler +from litellm.router import Router +from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler, RoutingArgs DEPLOYMENT_ID = "9876" KWARGS = { @@ -58,9 +59,9 @@ def test_sync_embedding_latency_is_json_serializable(): latencies = _recorded_latencies(cache) assert latencies, "expected a latency entry to be recorded" - assert all( - not isinstance(value, timedelta) for value in latencies - ), f"raw timedelta leaked into latency list: {latencies}" + assert all(not isinstance(value, timedelta) for value in latencies), ( + f"raw timedelta leaked into latency list: {latencies}" + ) assert latencies[-1] == pytest.approx(2.0) # the exact failure mode from production: redis cache sync json.dumps json.dumps({"latency": latencies}) @@ -84,9 +85,9 @@ async def test_async_embedding_latency_is_json_serializable(): latencies = _recorded_latencies(cache) assert latencies, "expected a latency entry to be recorded" - assert all( - not isinstance(value, timedelta) for value in latencies - ), f"raw timedelta leaked into latency list: {latencies}" + assert all(not isinstance(value, timedelta) for value in latencies), ( + f"raw timedelta leaked into latency list: {latencies}" + ) assert latencies[-1] == pytest.approx(3.0) json.dumps({"latency": latencies}) @@ -292,6 +293,85 @@ async def test_streaming_routing_ignores_per_token_ttft_samples_from_older_worke assert picked["model_info"]["id"] == FAST_TTFT_ID +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) +@pytest.mark.parametrize( + ("ttft_percentile", "first_samples", "second_samples", "expected_id"), + [ + (None, [0.1, 0.1, 1.0], [0.3, 0.3, 0.3], SLOW_TTFT_ID), + (0.5, [0.1, 0.1, 1.0], [0.3, 0.3, 0.3], FAST_TTFT_ID), + (0.9, [0.1, 0.1, 0.1, 0.1, 1.5], [0.3, 0.3, 0.3, 0.3, 0.3], SLOW_TTFT_ID), + ], + ids=["default_average", "p50", "p90"], +) +async def test_streaming_ttft_ranking_percentile( + sync_mode: bool, + ttft_percentile: float | None, + first_samples: list[float], + second_samples: list[float], + expected_id: str, +): + cache = DualCache() + routing_args = {} if ttft_percentile is None else {"ttft_percentile": ttft_percentile} + handler = LowestLatencyLoggingHandler(router_cache=cache, routing_args=routing_args) + cache.set_cache( + key=f"{MODEL_GROUP}_map", + value={ + FAST_TTFT_ID: {"time_to_first_token_seconds": first_samples}, + SLOW_TTFT_ID: {"time_to_first_token_seconds": second_samples}, + }, + ) + + if sync_mode: + picked = handler.get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=STREAMING_DEPLOYMENTS, + request_kwargs={"stream": True, "metadata": {}}, + ) + else: + picked = await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=STREAMING_DEPLOYMENTS, + request_kwargs={"stream": True, "metadata": {}}, + ) + + assert picked is not None + assert picked["model_info"]["id"] == expected_id + + +@pytest.mark.parametrize("ttft_percentile", [0, -0.1, 1.1]) +def test_ttft_percentile_validation(ttft_percentile: float): + with pytest.raises(ValidationError): + RoutingArgs(ttft_percentile=ttft_percentile) + + +@pytest.mark.parametrize("ttft_percentile", [0.5, 0.9, 0.95, 1.0]) +def test_ttft_percentile_accepts_valid_values(ttft_percentile: float): + assert RoutingArgs(ttft_percentile=ttft_percentile).ttft_percentile == ttft_percentile + + +@pytest.mark.asyncio +async def test_ttft_percentile_does_not_change_non_streaming_routing(): + cache = DualCache() + handler = LowestLatencyLoggingHandler(router_cache=cache, routing_args={"ttft_percentile": 0.9}) + cache.set_cache( + key=f"{MODEL_GROUP}_map", + value={ + FAST_TTFT_ID: {"latency": [1.0], "time_to_first_token_seconds": [0.1]}, + SLOW_TTFT_ID: {"latency": [0.2], "time_to_first_token_seconds": [1.5]}, + }, + ) + + picked = await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=STREAMING_DEPLOYMENTS, + request_kwargs={"stream": False, "metadata": {}}, + ) + + assert picked is not None + assert picked["model_info"]["id"] == SLOW_TTFT_ID + + @pytest.mark.asyncio @pytest.mark.parametrize( "cached_entry", @@ -318,3 +398,81 @@ async def test_async_get_available_deployments_treats_missing_samples_as_zero_la assert picked is not None assert picked["model_info"]["id"] == DEPLOYMENT_ID + + +def _latency_router(routing_strategy_args: dict) -> Router: + return Router( + model_list=[ + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": f"openai/{MODEL_GROUP}", "api_key": "sk-fake"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in (FAST_TTFT_ID, SLOW_TTFT_ID) + ], + routing_strategy="latency-based-routing", + routing_strategy_args=routing_strategy_args, + ) + + +def _seed_streaming_ttft(router: Router) -> None: + router.cache.set_cache( + key=f"{MODEL_GROUP}_map", + value={ + FAST_TTFT_ID: {"time_to_first_token_seconds": [0.1, 0.1, 1.0]}, + SLOW_TTFT_ID: {"time_to_first_token_seconds": [0.3, 0.3, 0.3]}, + }, + ) + + +async def _pick_streaming(router: Router) -> str: + picked = await router.async_get_available_deployment( + model=MODEL_GROUP, + request_kwargs={"stream": True, "metadata": {}}, + ) + return picked["model_info"]["id"] + + +@pytest.mark.asyncio +async def test_runtime_routing_strategy_args_update_applies_ttft_percentile(): + """A config reload that adds ttft_percentile must reach the live selector, + not sit unused until the proxy restarts.""" + router = _latency_router({"max_latency_list_size": 50}) + _seed_streaming_ttft(router) + + assert await _pick_streaming(router) == SLOW_TTFT_ID + + router.update_settings(routing_strategy_args={"max_latency_list_size": 50, "ttft_percentile": 0.5}) + + assert await _pick_streaming(router) == FAST_TTFT_ID + + +@pytest.mark.asyncio +async def test_runtime_routing_strategy_args_update_keeps_previous_args_when_invalid(): + router = _latency_router({"ttft_percentile": 0.5}) + _seed_streaming_ttft(router) + + router.update_settings(routing_strategy_args={"ttft_percentile": 5}) + + assert await _pick_streaming(router) == FAST_TTFT_ID + + +@pytest.mark.asyncio +async def test_runtime_routing_strategy_args_update_is_a_noop_without_a_selector(): + """simple-shuffle has no selector to re-link, so an args update must leave + the router alone instead of blowing up on a missing selector attribute.""" + router = Router( + model_list=[ + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": f"openai/{MODEL_GROUP}", "api_key": "sk-fake"}, + "model_info": {"id": FAST_TTFT_ID}, + } + ], + routing_strategy="simple-shuffle", + ) + + router.update_settings(routing_strategy_args={"ttl": 5}) + + assert router.routing_strategy_args == {"ttl": 5} + assert await _pick_streaming(router) == FAST_TTFT_ID From c7163a80ddfd34da743b06d770d60aba05b292eb Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:48:06 -0700 Subject: [PATCH 198/411] perf(proxy): collapse per-worker SGR upserts into one statement per flush (#40362) Each proxy worker flushed one Prisma upsert per active (date, category, route) bucket every interval, so the Postgres primary saw workers x routes statements per interval across the deployment. A flush now builds a single multi-row INSERT ... ON CONFLICT DO UPDATE, and with use_redis_transaction_buffer on the workers push snapshots to a Redis list that one lease-holding pod folds and commits, so the whole deployment costs one statement per interval. The leader keeps popping until the list is empty so a deployment wider than the dequeue cap cannot build a backlog, and rows that fail both the commit and the Redis re-queue fall back to the leader's own accumulator instead of being lost. Resolves LIT-7371 Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/proxy/db/gateway_request_tracking.py | 220 ++++++++-- litellm/proxy/proxy_server.py | 14 +- .../proxy/db/test_gateway_request_tracking.py | 391 +++++++++++++++--- 4 files changed, 537 insertions(+), 89 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 108a914e9c1..f9389d22dea 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -335,6 +335,7 @@ DEFAULT_SSL_CIPHERS: Final = os.getenv( ########### v2 Architecture constants for managing writing updates to the database ########### REDIS_UPDATE_BUFFER_KEY: Final = "litellm_spend_update_buffer" +REDIS_GATEWAY_REQUESTS_BUFFER_KEY: Final = "litellm_gateway_requests_buffer" REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_spend_update_buffer" REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_team_spend_update_buffer" REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_org_spend_update_buffer" diff --git a/litellm/proxy/db/gateway_request_tracking.py b/litellm/proxy/db/gateway_request_tracking.py index bebd74e877c..c9ace68db33 100644 --- a/litellm/proxy/db/gateway_request_tracking.py +++ b/litellm/proxy/db/gateway_request_tracking.py @@ -10,13 +10,28 @@ strings rather than passing the raw path through. Nothing a caller sends can add a key, so the fold and the table it commits to are bounded by (days x routes) however much traffic arrives, and the response path carries no unbounded queue that would block once full. + +A flush commits its whole snapshot as one multi-row ``INSERT ... ON CONFLICT DO +UPDATE`` rather than one upsert per key, so a worker costs the primary one +statement per interval however many routes it served. With +``use_redis_transaction_buffer`` on, workers instead push their snapshot to a +Redis list and one lock-holding pod folds every entry and writes the table, so +the deployment as a whole costs the primary one statement per interval. """ -from dataclasses import asdict +import json +from collections.abc import AsyncIterator, Iterable from datetime import datetime, timezone -from typing import TYPE_CHECKING, Final +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias + +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger +from litellm.caching import RedisCache +from litellm.constants import MAX_REDIS_BUFFER_DEQUEUE_COUNT, REDIS_GATEWAY_REQUESTS_BUFFER_KEY +from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.middleware.billable_request_metrics_middleware import BillableCategory from litellm.types.proxy.gateway_requests import ( GatewayRequestCounts, @@ -28,6 +43,15 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient _EMPTY: Final = GatewayRequestCounts(successful_requests=0, failed_requests=0) +_TABLE: Final = '"LiteLLM_DailyGatewayRequests"' +_COLUMNS_PER_ROW: Final = 5 +_UTC_NOW: Final = "(NOW() AT TIME ZONE 'UTC')" +GATEWAY_REQUESTS_JOB_NAME: Final = "update_gateway_requests_job" + +_BufferedRows: TypeAlias = tuple[tuple[str, str, str, int, int], ...] +_BUFFERED_ROWS: Final = TypeAdapter(_BufferedRows) +_BUFFERED_ENTRIES: Final = TypeAdapter(tuple[str | bytes, ...]) +_NO_COUNTS: Final[GatewayRequestSnapshot] = MappingProxyType({}) def _utc_date() -> str: @@ -59,20 +83,54 @@ class GatewayRequestAccumulator: route) however long the database is unreachable. This buys at-least-once, not exactly-once, and the cost is worth stating. - The batch commits inside its context manager's ``__aexit__``, so a failure - raised after the transaction committed (a connection dropped while reading - the acknowledgement) restores counts that are already persisted, and the - next flush increments them a second time. Exactly-once would need a dedup - key the upserts could ignore on replay. For a traffic-volume metric a rare + The statement commits on the server before its acknowledgement is read, so + a failure raised after the commit (a connection dropped while reading the + acknowledgement) restores counts that are already persisted, and the next + flush increments them a second time. Exactly-once would need a dedup key + the upsert could ignore on replay. For a traffic-volume metric a rare overcount on a dropped acknowledgement beats losing a whole interval to every database blip, so the trade is deliberate. """ - for key, counts in snapshot.items(): - existing = self._counts.get(key, _EMPTY) - self._counts[key] = GatewayRequestCounts( - successful_requests=existing.successful_requests + counts.successful_requests, - failed_requests=existing.failed_requests + counts.failed_requests, - ) + self._counts = dict(fold_counts(chain(self._counts.items(), snapshot.items()))) # mutable-ok: fold replaced + + +def fold_counts(items: Iterable[tuple[GatewayRequestKey, GatewayRequestCounts]]) -> GatewayRequestSnapshot: + """Sum counts key-wise; the result stays bounded by (date x category x route).""" + folded: Final[dict[GatewayRequestKey, GatewayRequestCounts]] = {} # mutable-ok: local fold returned once + for key, counts in items: + existing = folded.get(key, _EMPTY) + folded[key] = GatewayRequestCounts( + successful_requests=existing.successful_requests + counts.successful_requests, + failed_requests=existing.failed_requests + counts.failed_requests, + ) + return folded + + +def build_gateway_requests_upsert(snapshot: GatewayRequestSnapshot) -> tuple[str, tuple[str | int, ...]]: + """ + One ``INSERT ... ON CONFLICT DO UPDATE`` that increments every (date, category, + route) in the snapshot. Rows are ordered by the conflict key so concurrent + writers lock rows in the same order and cannot deadlock. + """ + ordered: Final = sorted(snapshot.items(), key=lambda item: (item[0].date, item[0].category, item[0].route)) + rows: Final = ", ".join( + f"(${base + 1}::text, ${base + 2}::text, ${base + 3}::text, ${base + 4}::bigint, ${base + 5}::bigint, {_UTC_NOW})" + for base in range(0, len(ordered) * _COLUMNS_PER_ROW, _COLUMNS_PER_ROW) + ) + sql: Final = ( + f'INSERT INTO {_TABLE} ("date", "category", "route", "successful_requests", "failed_requests", "updated_at")\n' + f"VALUES {rows}\n" + 'ON CONFLICT ("date", "category", "route") DO UPDATE SET\n' + f' "successful_requests" = {_TABLE}."successful_requests" + EXCLUDED."successful_requests",\n' + f' "failed_requests" = {_TABLE}."failed_requests" + EXCLUDED."failed_requests",\n' + f' "updated_at" = {_UTC_NOW}' + ) + params: Final[tuple[str | int, ...]] = tuple( + value + for key, counts in ordered + for value in (key.date, key.category, key.route, counts.successful_requests, counts.failed_requests) + ) + return sql, params async def commit_gateway_requests_to_db( @@ -80,50 +138,130 @@ async def commit_gateway_requests_to_db( prisma_client: "PrismaClient", snapshot: GatewayRequestSnapshot, ) -> None: - """Upsert one incrementing row per (date, category, route).""" + """Increment every (date, category, route) in the snapshot with a single statement.""" if not snapshot: return - ordered: Final = sorted(snapshot.items(), key=lambda item: (item[0].date, item[0].category, item[0].route)) + sql, params = build_gateway_requests_upsert(snapshot) + await prisma_client.db.execute_raw(sql, *params) # pyright: ignore[reportAny] # untyped prisma client - # pyright: ignore[reportAny] on both lines -- prisma's generated client is untyped, - # so .db and every table action off it resolve to Any at this boundary. The dict - # literals below are the shape prisma's generated inputs require. - async with prisma_client.db.batch_() as batcher: # pyright: ignore[reportAny] # untyped prisma client - for key, counts in ordered: - columns = asdict(key) - batcher.litellm_dailygatewayrequests.upsert( # pyright: ignore[reportAny] # untyped prisma client - where={"date_category_route": columns}, # mutable-ok: prisma input is dict-shaped - data={ # mutable-ok: prisma input is dict-shaped - "create": { # mutable-ok: prisma input is dict-shaped - **columns, - "successful_requests": counts.successful_requests, - "failed_requests": counts.failed_requests, - }, - "update": { # mutable-ok: prisma input is dict-shaped - "successful_requests": {"increment": counts.successful_requests}, # mutable-ok: as above - "failed_requests": {"increment": counts.failed_requests}, # mutable-ok: as above - }, - }, + verbose_proxy_logger.debug( + "Gateway request tracking - committed %d aggregated rows in one statement", len(snapshot) + ) + + +class GatewayRequestRedisBuffer: + """ + Folds every worker's snapshot through one Redis list so a single pod per + interval writes the table, mirroring the spend writer's transaction buffer. + + Each entry is one worker's snapshot as JSON rows; the lock holder pops them, + sums them, and commits one statement. A commit failure pushes the summed + rows back so the next holder retries, keeping the at-least-once guarantee. + If that push fails too, the rows go back to the holder's own accumulator so + they ride along with its next flush instead of vanishing with the pop. + """ + + def __init__(self, *, redis_cache: RedisCache, pod_lock_manager: PodLockManager) -> None: + self._redis_cache: Final = redis_cache + self._pod_lock_manager: Final = pod_lock_manager + + async def push(self, snapshot: GatewayRequestSnapshot) -> None: + if not snapshot: + return + rows: Final[_BufferedRows] = tuple( + (key.date, key.category, key.route, counts.successful_requests, counts.failed_requests) + for key, counts in snapshot.items() + ) + await self._redis_cache.async_rpush(key=REDIS_GATEWAY_REQUESTS_BUFFER_KEY, values=(json.dumps(rows),)) + + async def _pop_batch(self) -> tuple[str | bytes, ...]: + popped: Final[object] = await self._redis_cache.async_lpop( # pyright: ignore[reportAny] # redis returns Any + key=REDIS_GATEWAY_REQUESTS_BUFFER_KEY, count=MAX_REDIS_BUFFER_DEQUEUE_COUNT + ) + if not popped: + return () + return _BUFFERED_ENTRIES.validate_python(popped if isinstance(popped, list) else (popped,)) + + async def _pop_all(self) -> AsyncIterator[str | bytes]: + while True: + batch = await self._pop_batch() + for entry in batch: + yield entry + if len(batch) < MAX_REDIS_BUFFER_DEQUEUE_COUNT: + return + + async def pop(self) -> GatewayRequestSnapshot: + entries: Final = tuple([entry async for entry in self._pop_all()]) + return fold_counts( + ( + GatewayRequestKey(date=date, category=category, route=route), + GatewayRequestCounts(successful_requests=succeeded, failed_requests=failed), ) + for entry in entries + for date, category, route, succeeded, failed in _BUFFERED_ROWS.validate_json(entry) + ) - verbose_proxy_logger.debug("Gateway request tracking - committed %d aggregated rows", len(ordered)) + async def commit_if_leader(self, prisma_client: "PrismaClient") -> GatewayRequestSnapshot: + """ + Drain the list and write it as one statement, but only on the pod holding the job lock. + + The lock is a lease, never released: the holder re-enters it on every flush and + keeps committing alone until the TTL lapses, so the primary sees one statement + per flush interval deployment-wide instead of one per worker. + + Returns the popped rows that could be neither committed nor re-queued, for the + caller to keep in memory. Empty on success. + """ + if not await self._pod_lock_manager.acquire_lock(cronjob_id=GATEWAY_REQUESTS_JOB_NAME): + return _NO_COUNTS + buffered: Final = await self.pop() + try: + await commit_gateway_requests_to_db(prisma_client=prisma_client, snapshot=buffered) + except Exception: # noqa: BLE001 -- a failed commit must not stop the scheduler + verbose_proxy_logger.warning( + "Gateway request tracking - failed to commit %d buffered rows, re-queuing to Redis for the next flush", + len(buffered), + exc_info=True, + ) + return await self._requeue(buffered) + return _NO_COUNTS + + async def _requeue(self, snapshot: GatewayRequestSnapshot) -> GatewayRequestSnapshot: + try: + await self.push(snapshot) + except Exception: # noqa: BLE001 -- the rows go back to the caller's accumulator instead + verbose_proxy_logger.warning( + "Gateway request tracking - Redis re-queue failed, keeping %d rows in memory for the next flush", + len(snapshot), + exc_info=True, + ) + return snapshot + return _NO_COUNTS async def flush_gateway_requests( prisma_client: "PrismaClient", accumulator: GatewayRequestAccumulator, + redis_buffer: GatewayRequestRedisBuffer | None = None, ) -> None: """ Scheduler entrypoint. Never raises: a metering failure must not kill the job. + With ``redis_buffer`` the snapshot goes to Redis and only the lease holder + writes to Postgres. Shutdown passes no buffer so a departing worker writes its + own counts directly instead of parking them behind a lease it may not hold. + ``CancelledError`` is deliberately not caught, so a flush cancelled during shutdown drops its snapshot rather than restoring counts onto an accumulator the process is about to discard. """ snapshot: Final = accumulator.drain() try: - await commit_gateway_requests_to_db(prisma_client=prisma_client, snapshot=snapshot) + if redis_buffer is None: + await commit_gateway_requests_to_db(prisma_client=prisma_client, snapshot=snapshot) + else: + await redis_buffer.push(snapshot) except Exception: # noqa: BLE001 -- a failed flush must not stop the scheduler accumulator.restore(snapshot) verbose_proxy_logger.warning( @@ -131,3 +269,13 @@ async def flush_gateway_requests( len(snapshot), exc_info=True, ) + return + if redis_buffer is None: + return + try: + accumulator.restore(await redis_buffer.commit_if_leader(prisma_client)) + except Exception: # noqa: BLE001 -- entries still in Redis are drained by the next flush + verbose_proxy_logger.warning( + "Gateway request tracking - leader drain failed, buffered rows stay in Redis for the next flush", + exc_info=True, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c241e66049b..09e43eb74e1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -424,6 +424,7 @@ from litellm.proxy.db.exception_handler import ( ) from litellm.proxy.db.gateway_request_tracking import ( GatewayRequestAccumulator, + GatewayRequestRedisBuffer, flush_gateway_requests, ) from litellm.proxy.db.proxy_worker_heartbeat import ( @@ -2355,6 +2356,17 @@ open_telemetry_logger: OpenTelemetry | None = None gateway_request_accumulator: Final = GatewayRequestAccumulator() ### INITIALIZE GLOBAL LOGGING OBJECT ### proxy_logging_obj: ProxyLogging = ProxyLogging(user_api_key_cache=user_api_key_cache, premium_user=premium_user) + + +def _gateway_request_redis_buffer() -> GatewayRequestRedisBuffer | None: + """Shares the spend writer's transaction-buffer Redis and pod lock when use_redis_transaction_buffer is on.""" + writer: Final = proxy_logging_obj.db_spend_update_writer + redis_cache: Final = writer.redis_update_buffer.redis_cache + if redis_cache is None or not writer.redis_update_buffer._should_commit_spend_updates_to_redis(): + return None + return GatewayRequestRedisBuffer(redis_cache=redis_cache, pod_lock_manager=writer.pod_lock_manager) + + ### REDIS QUEUE ### async_result: Final = None celery_app_conn: Final = None @@ -9633,7 +9645,7 @@ class ProxyStartupEvent: flush_gateway_requests, "interval", seconds=batch_writing_interval, - args=(prisma_client, gateway_request_accumulator), + args=(prisma_client, gateway_request_accumulator, _gateway_request_redis_buffer()), id="update_gateway_requests_job", replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, diff --git a/tests/test_litellm/proxy/db/test_gateway_request_tracking.py b/tests/test_litellm/proxy/db/test_gateway_request_tracking.py index 93a11a914cb..045261e2d53 100644 --- a/tests/test_litellm/proxy/db/test_gateway_request_tracking.py +++ b/tests/test_litellm/proxy/db/test_gateway_request_tracking.py @@ -8,8 +8,11 @@ from datetime import datetime, timezone import pytest +from litellm.constants import MAX_REDIS_BUFFER_DEQUEUE_COUNT, REDIS_GATEWAY_REQUESTS_BUFFER_KEY from litellm.proxy.db.gateway_request_tracking import ( + GATEWAY_REQUESTS_JOB_NAME, GatewayRequestAccumulator, + GatewayRequestRedisBuffer, commit_gateway_requests_to_db, flush_gateway_requests, ) @@ -83,40 +86,47 @@ def test_drain_snapshot_is_not_mutated_by_later_records(): # ── commit ──────────────────────────────────────────────────────────────────── -class FakeTable: - def __init__(self) -> None: - self.upserts: list[dict] = [] - - def upsert(self, *, where: dict, data: dict) -> None: - self.upserts.append({"where": where, "data": data}) - - -class FakeBatcher: - def __init__(self, table: FakeTable) -> None: - self.litellm_dailygatewayrequests = table - - async def __aenter__(self) -> "FakeBatcher": - return self - - async def __aexit__(self, *args: object) -> bool: - return False - - class FakeDB: - def __init__(self, table: FakeTable) -> None: - self._table = table + def __init__(self) -> None: + self.statements: list[tuple[str, tuple[object, ...]]] = [] - def batch_(self) -> FakeBatcher: - return FakeBatcher(self._table) + async def execute_raw(self, query: str, *args: object) -> int: + self.statements.append((query, args)) + return len(args) // 5 class FakePrismaClient: def __init__(self) -> None: - self.table = FakeTable() - self.db = FakeDB(self.table) + self.db = FakeDB() -def test_commit_upserts_one_incrementing_row_per_key(): +def _rows_written(client: FakePrismaClient) -> list[tuple[object, ...]]: + """Every (date, category, route, successful, failed) tuple the database received, in statement order.""" + return [params[i : i + 5] for _, params in client.db.statements for i in range(0, len(params), 5)] + + +def test_commit_increments_with_a_single_statement_for_the_whole_snapshot(): + """One statement per flush is the whole point: the previous per-key upsert cost + the primary (workers x routes) statements per interval.""" + client = FakePrismaClient() + snapshot = { + GatewayRequestKey(date="2026-08-01", category="llm", route=route): ( + GatewayRequestCounts(successful_requests=7, failed_requests=2) + ) + for route in ("/chat/completions", "/embeddings", "/responses", "/v1/messages", "/mcp") + } + + asyncio.run(commit_gateway_requests_to_db(prisma_client=client, snapshot=snapshot)) + + assert len(client.db.statements) == 1 + sql, params = client.db.statements[0] + assert sql.count("ON CONFLICT") == 1 + assert sql.count("(NOW() AT TIME ZONE 'UTC'))") == 5 + assert len(params) == 25 + + +def test_commit_sql_adds_to_the_existing_row_instead_of_replacing_it(): + """A worker only knows its own share; the SQL must add EXCLUDED onto the stored count.""" client = FakePrismaClient() snapshot = { GatewayRequestKey(date="2026-08-01", category="llm", route="/chat/completions"): ( @@ -126,20 +136,36 @@ def test_commit_upserts_one_incrementing_row_per_key(): asyncio.run(commit_gateway_requests_to_db(prisma_client=client, snapshot=snapshot)) - assert len(client.table.upserts) == 1 - written = client.table.upserts[0] - assert written["where"] == { - "date_category_route": { - "date": "2026-08-01", - "category": "llm", - "route": "/chat/completions", - } + sql, params = client.db.statements[0] + assert 'INSERT INTO "LiteLLM_DailyGatewayRequests"' in sql + assert 'ON CONFLICT ("date", "category", "route") DO UPDATE SET' in sql + assert ( + '"successful_requests" = "LiteLLM_DailyGatewayRequests"."successful_requests" + EXCLUDED."successful_requests"' + in sql + ) + assert '"failed_requests" = "LiteLLM_DailyGatewayRequests"."failed_requests" + EXCLUDED."failed_requests"' in sql + assert params == ("2026-08-01", "llm", "/chat/completions", 7, 2) + + +def test_commit_placeholders_line_up_with_params(): + """$n positions are generated per row; a drift here silently swaps a route for a count.""" + client = FakePrismaClient() + snapshot = { + GatewayRequestKey(date="2026-08-01", category="llm", route="/chat/completions"): ( + GatewayRequestCounts(successful_requests=1, failed_requests=0) + ), + GatewayRequestKey(date="2026-08-01", category="mcp", route="/mcp"): ( + GatewayRequestCounts(successful_requests=0, failed_requests=3) + ), } - assert written["data"]["update"] == { - "successful_requests": {"increment": 7}, - "failed_requests": {"increment": 2}, - } - assert written["data"]["create"]["successful_requests"] == 7 + + asyncio.run(commit_gateway_requests_to_db(prisma_client=client, snapshot=snapshot)) + + sql, params = client.db.statements[0] + assert "($1::text, $2::text, $3::text, $4::bigint, $5::bigint," in sql + assert "($6::text, $7::text, $8::text, $9::bigint, $10::bigint," in sql + assert "$11" not in sql + assert params == ("2026-08-01", "llm", "/chat/completions", 1, 0, "2026-08-01", "mcp", "/mcp", 0, 3) def test_commit_is_deterministically_ordered(): @@ -154,17 +180,14 @@ def test_commit_is_deterministically_ordered(): asyncio.run(commit_gateway_requests_to_db(prisma_client=client, snapshot=snapshot)) - written_order = [ - (row["where"]["date_category_route"]["date"], row["where"]["date_category_route"]["category"]) - for row in client.table.upserts - ] + written_order = [(row[0], row[1]) for row in _rows_written(client)] assert written_order == [("2026-08-01", "llm"), ("2026-08-01", "mcp"), ("2026-08-02", "llm")] def test_commit_skips_the_database_entirely_when_nothing_accumulated(): client = FakePrismaClient() asyncio.run(commit_gateway_requests_to_db(prisma_client=client, snapshot={})) - assert client.table.upserts == [] + assert client.db.statements == [] # ── flush ───────────────────────────────────────────────────────────────────── @@ -177,12 +200,12 @@ def test_flush_drains_and_commits(): asyncio.run(flush_gateway_requests(client, acc)) - assert len(client.table.upserts) == 1 + assert len(client.db.statements) == 1 assert acc.drain() == {} class ExplodingDB: - def batch_(self): + async def execute_raw(self, query: str, *args: object) -> int: raise RuntimeError("db gone") @@ -208,10 +231,7 @@ def test_failed_flush_keeps_counts_for_the_next_attempt(): client = FakePrismaClient() asyncio.run(flush_gateway_requests(client, acc)) - assert client.table.upserts[0]["data"]["update"] == { - "successful_requests": {"increment": 1}, - "failed_requests": {"increment": 1}, - } + assert _rows_written(client) == [(_today(), "llm", "/chat/completions", 1, 1)] def test_restored_counts_merge_with_requests_recorded_meanwhile(): @@ -223,5 +243,272 @@ def test_restored_counts_merge_with_requests_recorded_meanwhile(): client = FakePrismaClient() asyncio.run(flush_gateway_requests(client, acc)) - assert len(client.table.upserts) == 1 - assert client.table.upserts[0]["data"]["update"]["successful_requests"] == {"increment": 2} + assert _rows_written(client) == [(_today(), "llm", "/chat/completions", 2, 0)] + + +class ExplodingDBWithInFlightRequest: + """Fails the write after a request has been recorded while it was in flight.""" + + def __init__(self, accumulator: GatewayRequestAccumulator) -> None: + self.accumulator = accumulator + + async def execute_raw(self, query: str, *args: object) -> int: + _record(self.accumulator, 500) + raise RuntimeError("db gone") + + +class ExplodingClientWithInFlightRequest: + def __init__(self, accumulator: GatewayRequestAccumulator) -> None: + self.db = ExplodingDBWithInFlightRequest(accumulator) + + +def test_restore_keeps_requests_recorded_while_the_failed_write_was_in_flight(): + acc = GatewayRequestAccumulator() + _record(acc, 200) + asyncio.run(flush_gateway_requests(ExplodingClientWithInFlightRequest(acc), acc)) + + client = FakePrismaClient() + asyncio.run(flush_gateway_requests(client, acc)) + + assert _rows_written(client) == [(_today(), "llm", "/chat/completions", 1, 1)] + + +# ── redis buffer ────────────────────────────────────────────────────────────── + + +class FakeRedis: + def __init__(self) -> None: + self.lists: dict[str, list[str]] = {} + + async def async_rpush(self, key: str, values: list[str]) -> int: + self.lists.setdefault(key, []).extend(values) + return len(self.lists[key]) + + async def async_lpop(self, key: str, count: int) -> list[str] | None: + queue = self.lists.get(key, []) + if not queue: + return None + popped, self.lists[key] = queue[:count], queue[count:] + return popped + + +class FakePodLock: + def __init__(self, *, leader: bool) -> None: + self.leader = leader + self.held: list[str] = [] + self.released: list[str] = [] + + async def acquire_lock(self, cronjob_id: str) -> bool: + self.held.append(cronjob_id) + return self.leader + + async def release_lock(self, cronjob_id: str) -> None: + self.released.append(cronjob_id) + + +class FakeLease: + """Redis-side view of the job lock: SET NX by pod id, re-entrant for the holder, freed only by release or TTL.""" + + def __init__(self) -> None: + self.holder: str | None = None + + +class FakeLeasePodLock: + def __init__(self, lease: FakeLease, pod_id: str) -> None: + self.lease = lease + self.pod_id = pod_id + + async def acquire_lock(self, cronjob_id: str) -> bool: + if self.lease.holder is None: + self.lease.holder = self.pod_id + return self.lease.holder == self.pod_id + + async def release_lock(self, cronjob_id: str) -> None: + if self.lease.holder == self.pod_id: + self.lease.holder = None + + +def _buffer(redis: FakeRedis, *, leader: bool) -> tuple[GatewayRequestRedisBuffer, FakePodLock]: + lock = FakePodLock(leader=leader) + return GatewayRequestRedisBuffer(redis_cache=redis, pod_lock_manager=lock), lock # pyright: ignore[reportArgumentType] # duck-typed fakes + + +def test_non_leader_workers_push_to_redis_and_never_touch_the_database(): + redis = FakeRedis() + client = FakePrismaClient() + for _ in range(3): + acc = GatewayRequestAccumulator() + _record(acc, 200) + buffer, _ = _buffer(redis, leader=False) + asyncio.run(flush_gateway_requests(client, acc, buffer)) + + assert client.db.statements == [] + assert len(redis.lists[REDIS_GATEWAY_REQUESTS_BUFFER_KEY]) == 3 + + +def test_leader_folds_every_workers_snapshot_into_one_statement(): + """Fifty workers each flushing the same routes must cost the primary one statement, not fifty.""" + redis = FakeRedis() + client = FakePrismaClient() + for _ in range(50): + acc = GatewayRequestAccumulator() + _record(acc, 200) + _record(acc, 500, route="/responses") + buffer, _ = _buffer(redis, leader=False) + asyncio.run(flush_gateway_requests(client, acc, buffer)) + + leader_acc = GatewayRequestAccumulator() + _record(leader_acc, 200) + leader, lock = _buffer(redis, leader=True) + asyncio.run(flush_gateway_requests(client, leader_acc, leader)) + + assert len(client.db.statements) == 1 + assert _rows_written(client) == [ + (_today(), "llm", "/chat/completions", 51, 0), + (_today(), "llm", "/responses", 0, 50), + ] + assert redis.lists[REDIS_GATEWAY_REQUESTS_BUFFER_KEY] == [] + assert lock.held == [GATEWAY_REQUESTS_JOB_NAME] + assert lock.released == [] + + +def test_leader_keeps_the_lease_so_staggered_pods_cost_one_statement_per_interval(): + """Pods flush on their own clocks; without the lease each one would win the lock in turn and commit alone.""" + redis = FakeRedis() + client = FakePrismaClient() + lease = FakeLease() + pods = tuple( + GatewayRequestRedisBuffer(redis_cache=redis, pod_lock_manager=FakeLeasePodLock(lease, f"pod-{i}")) # pyright: ignore[reportArgumentType] # duck-typed fakes + for i in range(4) + ) + + for _interval in range(3): + for pod in pods: + acc = GatewayRequestAccumulator() + _record(acc, 200) + asyncio.run(flush_gateway_requests(client, acc, pod)) + + assert lease.holder == "pod-0" + assert len(client.db.statements) == 3 + assert [row[3] for row in _rows_written(client)] == [1, 4, 4] + assert len(redis.lists[REDIS_GATEWAY_REQUESTS_BUFFER_KEY]) == 3 + + +def test_leader_drains_a_backlog_deeper_than_one_capped_pop(): + """More workers than MAX_REDIS_BUFFER_DEQUEUE_COUNT must not leave a growing tail queued behind the cap.""" + redis = FakeRedis() + client = FakePrismaClient() + workers = MAX_REDIS_BUFFER_DEQUEUE_COUNT * 2 + 1 + for _ in range(workers): + acc = GatewayRequestAccumulator() + _record(acc, 200) + buffer, _ = _buffer(redis, leader=False) + asyncio.run(flush_gateway_requests(client, acc, buffer)) + + leader, _ = _buffer(redis, leader=True) + asyncio.run(flush_gateway_requests(client, GatewayRequestAccumulator(), leader)) + + assert len(client.db.statements) == 1 + assert _rows_written(client) == [(_today(), "llm", "/chat/completions", workers, 0)] + assert redis.lists[REDIS_GATEWAY_REQUESTS_BUFFER_KEY] == [] + + +def test_leader_with_nothing_buffered_writes_nothing(): + redis = FakeRedis() + client = FakePrismaClient() + leader, lock = _buffer(redis, leader=True) + + asyncio.run(flush_gateway_requests(client, GatewayRequestAccumulator(), leader)) + + assert client.db.statements == [] + assert lock.released == [] + + +def test_leader_requeues_to_redis_when_the_database_commit_fails(): + """Counts popped from Redis are gone from every worker; a failed commit must put them back.""" + redis = FakeRedis() + acc = GatewayRequestAccumulator() + _record(acc, 200) + _record(acc, 200) + leader, lock = _buffer(redis, leader=True) + + asyncio.run(flush_gateway_requests(ExplodingClient(), acc, leader)) + + assert len(redis.lists[REDIS_GATEWAY_REQUESTS_BUFFER_KEY]) == 1 + assert lock.released == [] + assert acc.drain() == {} + + client = FakePrismaClient() + retry, _ = _buffer(redis, leader=True) + asyncio.run(flush_gateway_requests(client, GatewayRequestAccumulator(), retry)) + assert _rows_written(client) == [(_today(), "llm", "/chat/completions", 2, 0)] + + +class ExplodingRedis(FakeRedis): + async def async_rpush(self, key: str, values: list[str]) -> int: + raise RuntimeError("redis gone") + + +class UnreadableRedis(FakeRedis): + async def async_lpop(self, key: str, count: int) -> list[str] | None: + raise RuntimeError("redis gone mid-flush") + + +class UnwritableRedis(FakeRedis): + """Pops succeed, pushes fail: a Redis that went read-only between the leader's pop and its re-queue.""" + + async def async_rpush(self, key: str, values: list[str]) -> int: + raise RuntimeError("redis read-only") + + +def test_leader_keeps_popped_counts_in_memory_when_both_the_database_and_the_requeue_fail(): + """The pop removed the only copy; if Redis will not take it back the leader itself must carry it.""" + redis = FakeRedis() + worker_acc = GatewayRequestAccumulator() + _record(worker_acc, 200) + _record(worker_acc, 200) + worker, _ = _buffer(redis, leader=False) + asyncio.run(flush_gateway_requests(FakePrismaClient(), worker_acc, worker)) + + degraded = UnwritableRedis() + degraded.lists = redis.lists + leader_acc = GatewayRequestAccumulator() + leader, _ = _buffer(degraded, leader=True) + asyncio.run(flush_gateway_requests(ExplodingClient(), leader_acc, leader)) + assert degraded.lists[REDIS_GATEWAY_REQUESTS_BUFFER_KEY] == [] + + client = FakePrismaClient() + retry, _ = _buffer(redis, leader=True) + asyncio.run(flush_gateway_requests(client, leader_acc, retry)) + assert _rows_written(client) == [(_today(), "llm", "/chat/completions", 2, 0)] + + +def test_leader_whose_redis_read_fails_leaves_the_pushed_rows_for_the_next_flush(): + """The scheduler job must not raise, and nothing is popped so nothing needs restoring anywhere.""" + redis = UnreadableRedis() + acc = GatewayRequestAccumulator() + _record(acc, 200) + client = FakePrismaClient() + leader, _ = _buffer(redis, leader=True) + + asyncio.run(flush_gateway_requests(client, acc, leader)) + + assert client.db.statements == [] + assert acc.drain() == {} + assert len(redis.lists[REDIS_GATEWAY_REQUESTS_BUFFER_KEY]) == 1 + + +def test_failed_redis_push_keeps_counts_locally_for_the_next_flush(): + acc = GatewayRequestAccumulator() + _record(acc, 200) + _record(acc, 500) + buffer, lock = _buffer(ExplodingRedis(), leader=True) + + asyncio.run(flush_gateway_requests(FakePrismaClient(), acc, buffer)) + + assert lock.held == [] + assert acc.drain() == { + GatewayRequestKey(date=_today(), category="llm", route="/chat/completions"): ( + GatewayRequestCounts(successful_requests=1, failed_requests=1) + ) + } From 6195fbf5f3aaa656d916f0a7c3013ada34e87752 Mon Sep 17 00:00:00 2001 From: Animesh Kumar Date: Wed, 9 Sep 2026 23:24:12 +0530 Subject: [PATCH 199/411] test: isolate bedrock aws tests from ambient SSL env vars Nine cases assert the sts client is built with verify=True, but get_ssl_verify reads SSL_CERT_FILE and SSL_VERIFY, so the argument depended on the ambient environment. The published images set SSL_CERT_FILE, so the suite failed there while passing in CI. Fixes #40357 --- tests/test_litellm/llms/bedrock/test_base_aws_llm.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index f854d806bdc..a1c28f36e70 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -38,6 +38,14 @@ def flush_shared_bedrock_iam_cache(): yield +@pytest.fixture(autouse=True) +def _clean_ssl_env(monkeypatch): + """get_ssl_verify reads these, so the sts client's verify= would otherwise depend on + the ambient environment. The published images set SSL_CERT_FILE.""" + for env_var in ("SSL_CERT_FILE", "SSL_VERIFY"): + monkeypatch.delenv(env_var, raising=False) + + def test_base_aws_llm_instances_share_process_wide_iam_cache(): """Regression LIT-2662: new instances must reuse iam_cache (Bedrock passthrough is per-request).""" first = BaseAWSLLM() From c82c9cbcedf43661521270740273d6437d76c098 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 11:04:11 -0700 Subject: [PATCH 200/411] fix(router): strip encrypted reasoning on an auto-router tier change instead of a 503 (#40280) A Responses API follow-up that replays reasoning.encrypted_content is pinned to the deployment that minted it. Behind an auto-router the pre-routing hook rebinds the model to the tier it picked before the candidate pool is built, so a turn that classifies into a different tier never finds the origin and the affinity check raised its fail-fast 503, whose text claims a cooldown that does not exist When the deployment that minted the reasoning is not a member of the model group this turn is routed to, strip the encrypted reasoning (keeping any readable summary, string or block form) and dispatch to the routed group. Membership is tested by deployment id against the candidate set the router itself resolved for the route (routing group, model_name, team, and pattern alike), not by model-group name, so an alias, a provider-qualified spelling, a team-public name, or a pattern route of the same group is not misread as a tier change. An unknown origin (a removed deployment, or a forged/unauthenticated marker) is handled the same as a cross-group one and its reasoning is stripped, so a real cross-group id and a nonexistent id return the same response and cannot be used to enumerate deployment ids. Unavailability within the origin's own group keeps the existing 429/503 fail-fast, so the cooldown contract is unchanged Resolves LIT-7195 Claude-Session: https://claude.ai/code/session_01KAumQbhzk6jdWWHFLA8Jar Co-authored-by: Claude Opus 4.8 --- litellm/responses/utils.py | 43 ++ litellm/router.py | 34 ++ .../encrypted_content_affinity_check.py | 73 ++- .../test_encrypted_content_affinity_check.py | 561 ++++++++++++------ tests/test_litellm/test_router.py | 42 ++ 5 files changed, 557 insertions(+), 196 deletions(-) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 540d492beec..599e978df6a 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -543,6 +543,49 @@ class ResponsesAPIRequestUtils: return request_input + @staticmethod + def strip_encrypted_reasoning_from_input(request_input: object) -> None: + """Drop reasoning items the routed deployment cannot decrypt, keeping their readable summary. + + Mutates ``request_input`` in place: the router's fallback snapshot shares this + list object, so a rebound list would replay the stripped items on the fallback hop. + """ + if not isinstance(request_input, list): + return + items: Final = cast(list[object], request_input) # cast-ok: untyped client json + stripped: Final = tuple(ResponsesAPIRequestUtils._without_encrypted_reasoning(item) for item in items) + items[:] = (item for item in stripped if item is not None) # rebind-ok: list shared with fallback snapshot + + @staticmethod + def _without_encrypted_reasoning(item: object) -> object | None: + if not isinstance(item, dict): + return item + reasoning: Final = cast(Mapping[str, object], item) # cast-ok: untyped client json + if reasoning.get("type") != "reasoning" or not reasoning.get("encrypted_content"): + return reasoning + readable: Final = any( + ResponsesAPIRequestUtils._has_readable_text(reasoning.get(key)) for key in ("summary", "content") + ) + if not readable: + return None + kept: Final[dict[str, object]] = { # mutable-ok: request item rebuilt without the undecryptable keys + key: value for key, value in reasoning.items() if key not in ("encrypted_content", "id") + } + return kept + + @staticmethod + def _has_readable_text(value: object) -> bool: + """A reasoning item's ``summary``/``content`` carries readable text: a non-empty string, or a + list holding at least one block with a non-empty ``text`` field (summary_text / output_text).""" + if isinstance(value, str): + return bool(value.strip()) + if isinstance(value, list): + return any( + isinstance(block, dict) and bool(cast(Mapping[str, object], block).get("text")) # cast-ok: untyped json + for block in value + ) + return False + @staticmethod def _build_responses_api_response_id( custom_llm_provider: str | None, diff --git a/litellm/router.py b/litellm/router.py index 2fbdf541487..68ace283949 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11167,6 +11167,40 @@ class Router: return ids + def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: + """ + Deployment ids that could serve ``model`` for ``team_id``, unioned across the paths + the router resolves a route through: ``model_group_alias``, a routing group, the + ``model_name`` and team indexes, and wildcard pattern routes. Read-only and + side-effect-free, unlike ``_common_checks_available_deployment`` which also applies + fallbacks and can raise. Lets a pre-call check tell a genuine cross-group route from + same-group unavailability without re-deriving that precedence at the call site, and + without leaking deployment ids into request kwargs bound for the provider. + """ + resolved: Final = self._get_model_from_alias(model=model) or model + routing_group_members: Final = self._get_routing_group_deployments(model=resolved, team_id=team_id) + if routing_group_members is not None: + return self._deployment_ids(routing_group_members) + if resolved in self.model_names: + return self._deployment_ids(self._get_all_deployments(model_name=resolved, team_id=team_id)) + team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None + return self._deployment_ids( + ( + *self._get_all_deployments(model_name=resolved, team_id=team_id), + *(self.pattern_router.route(resolved) or ()), + *((team_router.route(resolved) or ()) if team_router is not None else ()), + ) + ) + + @staticmethod + def _deployment_ids(deployments: Sequence[Mapping[str, object]]) -> frozenset[str]: + return frozenset( + str(model_info["id"]) + for deployment in deployments + for model_info in (deployment.get("model_info"),) + if isinstance(model_info, Mapping) and model_info.get("id") is not None + ) + def has_model_id(self, candidate_id: str) -> bool: """ O(1) membership check for a deployment ID without allocating large lists. diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index b623e31ce06..d10153881d9 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -37,13 +37,13 @@ Safe to enable globally: """ import time +from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Optional, Protocol, cast import httpx from litellm._logging import verbose_router_logger from litellm.exceptions import ( - BadRequestError, RateLimitError, ServiceUnavailableError, ) @@ -158,6 +158,23 @@ class EncryptedContentAffinityCheck(CustomLogger): return deployment return None + @staticmethod + def _request_team_id(request_kwargs: Mapping[str, object]) -> str | None: + containers: Final = (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")) + team_ids: Final = (c.get("user_api_key_team_id") for c in containers if isinstance(c, Mapping)) + return next((tid for tid in team_ids if isinstance(tid, str)), None) + + def _routed_group_candidate_model_ids(self, request_kwargs: Mapping[str, object], model: str) -> frozenset[str]: + """ + Deployment ids that could serve this turn's routed ``model``, as the router + resolves a route (model_group_alias / routing group / model_name / team / + pattern). Delegates to the router so the full precedence is not re-derived here + and no deployment ids are written into request kwargs bound for the provider. + """ + if self.router is None: + return frozenset() + return self.router.get_candidate_model_ids_for_route(model=model, team_id=self._request_team_id(request_kwargs)) + @staticmethod def _encryption_boundary_key( litellm_params: object, @@ -225,10 +242,14 @@ class EncryptedContentAffinityCheck(CustomLogger): """ If the request ``input`` contains litellm-encoded item IDs, decode the embedded ``model_id`` and pin the request to that deployment. Raises - ``RateLimitError`` / ``ServiceUnavailableError`` / ``BadRequestError`` - when the originating deployment is unavailable and no encryption-boundary - peer exists, rather than dispatching a doomed request to a non-peer - deployment. The 429/503 split mirrors the originating cooldown's status: + ``RateLimitError`` / ``ServiceUnavailableError`` when the originating + deployment is a member of the routed model group but currently unavailable + and no encryption-boundary peer exists, rather than dispatching a doomed + request to a non-peer deployment. When the origin is not a member of the + routed group (an auto-router tier change, a model switch with no peer, a + removed deployment, or an unknown/forged marker), the encrypted reasoning is + stripped and the request dispatches with its readable history instead. The + 429/503 split mirrors the originating cooldown's status: a 429-induced cooldown surfaces as 429 (with ``Retry-After`` set to the remaining cooldown window) so OpenAI-compatible clients back off and retry after the deployment is eligible again. @@ -285,12 +306,34 @@ class EncryptedContentAffinityCheck(CustomLogger): request_kwargs["_encrypted_content_affinity_pinned"] = True return boundary_matches - # Dispatching to a non-peer would guarantee an upstream - # `invalid_encrypted_content` 400, so fail fast with a clearer error. + # The origin cannot serve this turn's routed group and no peer shares the boundary, so its + # encrypted reasoning can never decrypt here. Strip it, keep the readable history, and dispatch + # to the routed group instead of failing. Membership is tested by deployment id against the set + # the router actually resolved for this route, not by model-group name, so an alias, a + # provider-qualified spelling, a team-public name, or a pattern route of the same group is not + # mistaken for a tier change. An unknown origin (a removed deployment, or a forged marker) is + # treated the same as a cross-group one, which also denies an authenticated caller a + # deployment-id existence oracle: a real cross-group id and a nonexistent id both strip and + # dispatch rather than returning distinguishable responses. Only a genuine same-group member + # that is currently unavailable falls through to the fail-fast, preserving the cooldown contract. + routed_group_model_ids: Final = ( + self._routed_group_candidate_model_ids(request_kwargs, model) if originating is not None else frozenset() + ) + if str(model_id) not in routed_group_model_ids: + verbose_router_logger.debug( + "EncryptedContentAffinityCheck: model_id=%s is not a candidate for the routed group %s; " + "forwarding without its encrypted reasoning", + model_id, + model, + ) + ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input) + return typed_healthy_deployments + + # The origin is a member of the routed group but currently unavailable (cooled down); fail fast + # rather than dispatching to a non-peer, which would guarantee an upstream 400. raise await self._unavailable_origin_error( model=model, model_id=model_id, - originating=originating, parent_otel_span=parent_otel_span, ) @@ -298,25 +341,11 @@ class EncryptedContentAffinityCheck(CustomLogger): self, model: str, model_id: str, - originating: Deployment | None, parent_otel_span: Span | None, ) -> Exception: # Public error messages intentionally omit the originating ``model_id`` so # an authenticated caller forging encrypted-content markers cannot use the # error surface to enumerate which deployment IDs exist on this router. - if originating is None: - return BadRequestError( - message=( - "The deployment that produced this encrypted_content is no " - "longer configured on this router, and no deployment on the " - "same encryption boundary is available. Re-issue the request " - "without the stale encrypted_content items, or restore the " - "originating deployment." - ), - model=model, - llm_provider="", - ) - cooldown: Final = await self._get_origin_cooldown(model_id=model_id, parent_otel_span=parent_otel_span) if cooldown is not None and str(cooldown.get("status_code")) == "429": diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index dac991a41c4..3961c8d74c2 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -16,12 +16,10 @@ The mechanism works without any cache and supports two encoding strategies: """ import time -from typing import List, Optional from unittest.mock import AsyncMock, patch import pytest - import litellm from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponsesAPIResponse @@ -68,9 +66,7 @@ class TestEncryptedItemIdCodec: def test_roundtrip(self): model_id = "deployment-1" original_item_id = "rs_abc123def456" - encoded = ResponsesAPIRequestUtils._build_encrypted_item_id( - model_id, original_item_id - ) + encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id) assert encoded.startswith("encitem_") decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded) assert decoded is not None @@ -81,9 +77,7 @@ class TestEncryptedItemIdCodec: """Decoding must succeed even if base64 padding (=) was stripped in transit.""" model_id = "gpt-5.1-codex-openai-2" original_item_id = "rs_0efb96cb222403210069a01d5d52588196a9dc394ffdb89d00" - encoded = ResponsesAPIRequestUtils._build_encrypted_item_id( - model_id, original_item_id - ) + encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id) # Strip any trailing '=' to simulate what happens in transit stripped = encoded.rstrip("=") decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(stripped) @@ -100,9 +94,7 @@ class TestEncryptedItemIdCodec: """item_id values containing ';' must survive the roundtrip.""" model_id = "deployment-1" original_item_id = "rs_part1;part2;part3" - encoded = ResponsesAPIRequestUtils._build_encrypted_item_id( - model_id, original_item_id - ) + encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id) decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded) assert decoded is not None assert decoded["item_id"] == original_item_id @@ -118,11 +110,7 @@ class TestUpdateEncryptedContentItemIds: {"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"}, ], } - result = ( - ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response( - response, model_id - ) - ) + result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, model_id) # Plain message item untouched assert result["output"][0]["id"] == "msg_abc" # Reasoning item with encrypted_content gets encoded @@ -133,16 +121,8 @@ class TestUpdateEncryptedContentItemIds: assert decoded["item_id"] == "rs_xyz" def test_no_op_when_model_id_is_none(self): - response = { - "output": [ - {"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"} - ] - } - result = ( - ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response( - response, None - ) - ) + response = {"output": [{"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"}]} + result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, None) assert result["output"][0]["id"] == "rs_xyz" @@ -151,9 +131,7 @@ class TestEncryptedContentWrapping: """Test wrapping encrypted_content with model_id metadata.""" model_id = "deployment-1" original_content = "gAAAAABpnW_yEYmSNEyOG_original_encrypted_data" - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_content, model_id - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id) assert wrapped.startswith("litellm_enc:") assert wrapped != original_content @@ -170,9 +148,7 @@ class TestEncryptedContentWrapping: ( model_id, content, - ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id( - plain_content - ) + ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(plain_content) assert model_id is None assert content == plain_content @@ -189,11 +165,7 @@ class TestEncryptedContentWrapping: }, ], } - result = ( - ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response( - response, model_id - ) - ) + result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, model_id) assert result["output"][0].get("encrypted_content") is None wrapped = result["output"][1]["encrypted_content"] assert wrapped.startswith("litellm_enc:") @@ -210,19 +182,13 @@ class TestRestoreEncryptedContentItemIds: def test_restores_encoded_ids(self): model_id = "deployment-1" original_id = "rs_encrypted_item_456" - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - model_id, original_id - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_id) request_input = [ {"type": "message", "id": "msg_abc123", "role": "assistant"}, {"type": "reasoning", "id": encoded_id, "encrypted_content": "secret"}, ] - restored = ( - ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input( - request_input - ) - ) + restored = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(request_input) assert restored[0]["id"] == "msg_abc123" assert restored[1]["id"] == original_id @@ -230,33 +196,21 @@ class TestRestoreEncryptedContentItemIds: """Test that wrapped encrypted_content is unwrapped before forwarding.""" model_id = "deployment-1" original_content = "gAAAAABpnW_yEYmSNEyOG_original" - wrapped_content = ( - ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_content, model_id - ) - ) + wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id) request_input = [ {"type": "reasoning", "encrypted_content": wrapped_content}, ] - restored = ( - ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input( - request_input - ) - ) + restored = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(request_input) assert restored[0]["encrypted_content"] == original_content def test_no_op_for_plain_string_input(self): - result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input( - "Hello world" - ) + result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input("Hello world") assert result == "Hello world" def test_no_op_for_unencoded_ids(self): request_input = [{"type": "message", "id": "msg_plain"}] - result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input( - request_input - ) + result = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(request_input) assert result[0]["id"] == "msg_plain" @@ -283,9 +237,7 @@ async def test_encrypted_content_affinity_tracks_and_routes(): "id": "msg_abc123", "status": "completed", "role": "assistant", - "content": [ - {"type": "output_text", "text": "Hello!", "annotations": []} - ], + "content": [{"type": "output_text", "text": "Hello!", "annotations": []}], }, { "type": "reasoning", @@ -347,9 +299,9 @@ async def test_encrypted_content_affinity_tracks_and_routes(): # The response must have rewritten the encrypted item's ID to encoded form encoded_item_id = _extract_encoded_item_id(first_response) - assert encoded_item_id.startswith( - "encitem_" - ), f"Expected output item ID to be rewritten to encitem_... but got {encoded_item_id!r}" + assert encoded_item_id.startswith("encitem_"), ( + f"Expected output item ID to be rewritten to encitem_... but got {encoded_item_id!r}" + ) # Verify the encoded ID decodes back to the correct deployment + original ID decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded_item_id) @@ -371,9 +323,9 @@ async def test_encrypted_content_affinity_tracks_and_routes(): ) second_model_id = second_response._hidden_params["model_id"] - assert ( - second_model_id == first_model_id - ), f"Expected affinity to route to {first_model_id}, but got {second_model_id}" + assert second_model_id == first_model_id, ( + f"Expected affinity to route to {first_model_id}, but got {second_model_id}" + ) @pytest.mark.asyncio @@ -478,9 +430,7 @@ async def test_encrypted_content_affinity_bypasses_rpm_limits(): # Extract encoded item ID from the first response output encoded_item_id = _extract_encoded_item_id(first_response) - assert encoded_item_id.startswith( - "encitem_" - ), f"Expected encitem_... but got {encoded_item_id!r}" + assert encoded_item_id.startswith("encitem_"), f"Expected encitem_... but got {encoded_item_id!r}" # Follow-up with the encoded item ID — should pin to same deployment second_response = await router.aresponses( @@ -628,17 +578,13 @@ async def test_encrypted_content_affinity_with_wrapped_content_no_id(): if hasattr(first_item, "encrypted_content") else first_item.get("encrypted_content") ) - assert wrapped_content.startswith( - "litellm_enc:" - ), f"Expected wrapped content but got {wrapped_content[:50]}..." + assert wrapped_content.startswith("litellm_enc:"), f"Expected wrapped content but got {wrapped_content[:50]}..." # Verify we can extract model_id from wrapped content ( extracted_model_id, _, - ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id( - wrapped_content - ) + ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped_content) assert extracted_model_id == first_model_id # Second request: use wrapped encrypted_content WITHOUT an ID (Codex behavior) @@ -653,9 +599,9 @@ async def test_encrypted_content_affinity_with_wrapped_content_no_id(): ) second_model_id = second_response._hidden_params["model_id"] - assert ( - second_model_id == first_model_id - ), f"Expected affinity to route to {first_model_id}, but got {second_model_id}" + assert second_model_id == first_model_id, ( + f"Expected affinity to route to {first_model_id}, but got {second_model_id}" + ) def test_encrypted_content_wrapping_preserves_original_content(): @@ -664,13 +610,9 @@ def test_encrypted_content_wrapping_preserves_original_content(): This is critical for streaming responses where content must round-trip correctly. """ model_id = "test-deployment-1" - original_encrypted_content = ( - "gAAAAABpnW_yEYmSNEyOG_streaming_test_content_with_special_chars==+/" - ) + original_encrypted_content = "gAAAAABpnW_yEYmSNEyOG_streaming_test_content_with_special_chars==+/" - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_encrypted_content, model_id - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_encrypted_content, model_id) assert wrapped.startswith("litellm_enc:") assert wrapped != original_encrypted_content @@ -691,9 +633,7 @@ def test_encrypted_content_wrapping_with_multiple_semicolons(): model_id = "deployment-with-semicolons" original_content = "gAAAAAB;some;content;with;semicolons" - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_content, model_id - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id) ( extracted_model_id, @@ -764,9 +704,7 @@ async def test_encrypted_content_affinity_preserves_litellm_metadata_for_respons request_kwargs=request_kwargs, ) - assert ( - request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True - ) + assert request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True assert request_kwargs["litellm_metadata"]["model_info"] == {"id": "dep-1"} @@ -777,9 +715,7 @@ def test_encrypted_content_wrapping_empty_string(): model_id = "test-deployment" original_content = "" - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( - original_content, model_id - ) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(original_content, model_id) assert wrapped.startswith("litellm_enc:") @@ -1132,9 +1068,7 @@ def test_boundary_key_accepts_pydantic_litellm_params_instance(): "api_key": "fake-azure-resource-key-a", } - pydantic_key = EncryptedContentAffinityCheck._encryption_boundary_key( - pydantic_params - ) + pydantic_key = EncryptedContentAffinityCheck._encryption_boundary_key(pydantic_params) plain_key = EncryptedContentAffinityCheck._encryption_boundary_key(plain_params) assert pydantic_key is not None @@ -1161,18 +1095,8 @@ def test_boundary_key_rejects_non_dict_like_inputs(): for bad in (None, [], "not a dict", 42, object()): assert EncryptedContentAffinityCheck._encryption_boundary_key(bad) is None - assert ( - EncryptedContentAffinityCheck._encryption_boundary_key( - {"api_base": "", "api_key": "k"} - ) - is None - ) - assert ( - EncryptedContentAffinityCheck._encryption_boundary_key( - {"api_base": "https://x"} - ) - is None - ) + assert EncryptedContentAffinityCheck._encryption_boundary_key({"api_base": "", "api_key": "k"}) is None + assert EncryptedContentAffinityCheck._encryption_boundary_key({"api_base": "https://x"}) is None # --------------------------------------------------------------------------- @@ -1180,10 +1104,11 @@ def test_boundary_key_rejects_non_dict_like_inputs(): # --------------------------------------------------------------------------- -def _make_originating_mock(api_base: str, api_key: str): +def _make_originating_mock(api_base: str, api_key: str, model_name: str = "gpt-5.4"): from unittest.mock import MagicMock originating = MagicMock() + originating.model_name = model_name originating.litellm_params.model_dump.return_value = { "api_base": api_base, "api_key": api_key, @@ -1192,19 +1117,23 @@ def _make_originating_mock(api_base: str, api_key: str): def _make_router_mock_with_cooldown( - originating, cooldown_entries: Optional[List[tuple]] = None + originating, + cooldown_entries: list[tuple] | None = None, + routed_group_model_ids: list[str] | None = None, ): """ Build a MagicMock router whose ``cooldown_cache.async_get_active_cooldowns`` - returns ``cooldown_entries`` (defaulting to ``[]`` — no active cooldown). + returns ``cooldown_entries`` (defaulting to ``[]`` — no active cooldown), and + whose ``get_candidate_model_ids_for_route`` returns ``routed_group_model_ids`` + (the deployment ids the router resolves for the routed model; defaulting to ``[]`` + — origin absent from the routed group, i.e. a tier change). """ from unittest.mock import AsyncMock, MagicMock mock_router = MagicMock() mock_router.get_deployment.return_value = originating - mock_router.cooldown_cache.async_get_active_cooldowns = AsyncMock( - return_value=list(cooldown_entries or []) - ) + mock_router.cooldown_cache.async_get_active_cooldowns = AsyncMock(return_value=list(cooldown_entries or [])) + mock_router.get_candidate_model_ids_for_route.return_value = frozenset(routed_group_model_ids or []) return mock_router @@ -1235,15 +1164,15 @@ async def test_affinity_raises_service_unavailable_when_origin_cooled_for_non_42 }, ) ], + routed_group_model_ids=["deployment-a-cooled", "deployment-b"], ) check = EncryptedContentAffinityCheck(router=mock_router) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-a-cooled", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a-cooled", "rs_test") healthy_only_b = [ { "model_info": {"id": "deployment-b"}, + "model_name": "gpt-5.4", "litellm_params": { "api_base": "https://account-b.openai.azure.com/", "api_key": "key-b", @@ -1297,15 +1226,15 @@ async def test_affinity_raises_rate_limit_with_retry_after_when_origin_cooled_fo }, ) ], + routed_group_model_ids=["deployment-a-cooled-429", "deployment-b"], ) check = EncryptedContentAffinityCheck(router=mock_router) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-a-cooled-429", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a-cooled-429", "rs_test") healthy_only_b = [ { "model_info": {"id": "deployment-b"}, + "model_name": "gpt-5.4", "litellm_params": { "api_base": "https://account-b.openai.azure.com/", "api_key": "key-b", @@ -1345,15 +1274,16 @@ async def test_affinity_raises_service_unavailable_when_origin_filtered_without_ ) originating = _make_originating_mock("https://account-a.openai.azure.com/", "key-a") - mock_router = _make_router_mock_with_cooldown(originating, cooldown_entries=[]) + mock_router = _make_router_mock_with_cooldown( + originating, cooldown_entries=[], routed_group_model_ids=["deployment-a-filtered", "deployment-b"] + ) check = EncryptedContentAffinityCheck(router=mock_router) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-a-filtered", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a-filtered", "rs_test") healthy_only_b = [ { "model_info": {"id": "deployment-b"}, + "model_name": "gpt-5.4", "litellm_params": { "api_base": "https://account-b.openai.azure.com/", "api_key": "key-b", @@ -1377,15 +1307,18 @@ async def test_affinity_raises_service_unavailable_when_origin_filtered_without_ @pytest.mark.asyncio -async def test_affinity_raises_bad_request_when_origin_removed(): +async def test_affinity_strips_and_dispatches_when_origin_is_unknown_or_removed(): """ - Originating deployment was removed from the router config and no boundary - peer is available. This is permanent (the stale encrypted_content cannot - be honored), so surface a 400 with actionable text. + A removed deployment, or a forged/unknown affinity marker, resolves to no + originating deployment. It is handled like a cross-group origin: the encrypted + reasoning is stripped and the request dispatches with its readable history, + rather than returning a distinguishable error. That uniform handling denies an + authenticated caller a deployment-id existence oracle, an existing cross-group id + and a nonexistent id both strip and proceed, so responses cannot be told apart. + The membership lookup is skipped entirely when the origin is unknown. """ from unittest.mock import MagicMock - from litellm.exceptions import BadRequestError from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( EncryptedContentAffinityCheck, ) @@ -1394,12 +1327,11 @@ async def test_affinity_raises_bad_request_when_origin_removed(): mock_router.get_deployment.return_value = None check = EncryptedContentAffinityCheck(router=mock_router) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-removed", "rs_test" - ) - healthy_only_b = [ + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-removed") + routed_pool = [ { "model_info": {"id": "deployment-b"}, + "model_name": "gpt-5.4", "litellm_params": { "api_base": "https://account-b.openai.azure.com/", "api_key": "key-b", @@ -1408,18 +1340,28 @@ async def test_affinity_raises_bad_request_when_origin_removed(): } ] request_kwargs = { - "input": [{"id": encoded_id, "type": "reasoning"}], + "litellm_metadata": {}, + "input": [ + {"role": "user", "content": "why is the sky blue?"}, + { + "type": "reasoning", + "encrypted_content": wrapped, + "summary": [{"type": "summary_text", "text": "scattering"}], + }, + {"role": "user", "content": "and sunsets?"}, + ], } - with pytest.raises(BadRequestError) as excinfo: - await check.async_filter_deployments( - model="gpt-5.4", - healthy_deployments=healthy_only_b, - messages=None, - request_kwargs=request_kwargs, - ) + result = await check.async_filter_deployments( + model="gpt-5.4", + healthy_deployments=routed_pool, + messages=None, + request_kwargs=request_kwargs, + ) - assert "deployment-removed" not in str(excinfo.value) + assert result is routed_pool + assert not any(isinstance(item, dict) and item.get("encrypted_content") for item in request_kwargs["input"]) + mock_router.get_candidate_model_ids_for_route.assert_not_called() @pytest.mark.asyncio @@ -1444,9 +1386,7 @@ async def test_affinity_does_not_raise_when_boundary_peer_available(): mock_router.get_deployment.return_value = originating check = EncryptedContentAffinityCheck(router=mock_router) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-a", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a", "rs_test") peer = { "model_info": {"id": "deployment-a-peer"}, "litellm_params": { @@ -1490,9 +1430,7 @@ async def test_model_group_affinity_config_enables_encrypted_content_affinity(): }, target_deployment, ] - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-b", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-b", "rs_test") request_kwargs = { "input": [{"type": "reasoning", "id": encoded_id}], "litellm_metadata": {}, @@ -1536,9 +1474,7 @@ async def test_model_group_affinity_config_does_not_disable_global_encrypted_con }, target_deployment, ] - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-b", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-b", "rs_test") request_kwargs = { "input": [{"type": "reasoning", "id": encoded_id}], "litellm_metadata": {}, @@ -1600,15 +1536,9 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen try: callbacks = router.optional_callbacks or [] - deployment_callback = next( - cb for cb in callbacks if isinstance(cb, DeploymentAffinityCheck) - ) - encrypted_content_callback = next( - cb for cb in callbacks if isinstance(cb, EncryptedContentAffinityCheck) - ) - assert callbacks.index(encrypted_content_callback) < callbacks.index( - deployment_callback - ) + deployment_callback = next(cb for cb in callbacks if isinstance(cb, DeploymentAffinityCheck)) + encrypted_content_callback = next(cb for cb in callbacks if isinstance(cb, EncryptedContentAffinityCheck)) + assert callbacks.index(encrypted_content_callback) < callbacks.index(deployment_callback) assert encrypted_content_callback.enable_global_affinity is False cache_key = DeploymentAffinityCheck.get_affinity_cache_key( @@ -1620,9 +1550,7 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen value={"model_id": "deployment-a"}, ttl=60, ) - encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id( - "deployment-b", "rs_test" - ) + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-b", "rs_test") request_kwargs = { "input": [ { @@ -1643,16 +1571,301 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen ) assert after_deployment_affinity == [deployment_a, deployment_b] - after_encrypted_content_affinity = ( - await encrypted_content_callback.async_filter_deployments( - model=model_group, - healthy_deployments=after_deployment_affinity, - messages=None, - request_kwargs=request_kwargs, - ) + after_encrypted_content_affinity = await encrypted_content_callback.async_filter_deployments( + model=model_group, + healthy_deployments=after_deployment_affinity, + messages=None, + request_kwargs=request_kwargs, ) assert after_encrypted_content_affinity == [deployment_b] assert request_kwargs.get("_encrypted_content_affinity_pinned") is True finally: router.discard() + + +class TestStripEncryptedReasoningFromInput: + def test_keeps_summary_and_drops_encrypted_content_and_id(self): + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a") + encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-a", "rs_1") + request_input = [ + {"role": "user", "content": "first turn"}, + { + "type": "reasoning", + "id": encoded_id, + "encrypted_content": wrapped, + "summary": [{"type": "summary_text", "text": "thought about it"}], + }, + {"type": "reasoning", "id": encoded_id, "encrypted_content": wrapped}, + {"type": "reasoning", "encrypted_content": wrapped, "summary": []}, + {"type": "message", "id": "msg_1", "role": "assistant", "content": "hi"}, + {"role": "user", "content": "second turn"}, + ] + ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input) + assert request_input == [ + {"role": "user", "content": "first turn"}, + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "thought about it"}], + }, + {"type": "message", "id": "msg_1", "role": "assistant", "content": "hi"}, + {"role": "user", "content": "second turn"}, + ] + + def test_keeps_string_form_summary_when_stripping(self): + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a") + request_input = [ + {"type": "reasoning", "encrypted_content": wrapped, "summary": "plain string thought"}, + { + "type": "reasoning", + "encrypted_content": wrapped, + "content": [{"type": "output_text", "text": "in content"}], + }, + {"type": "reasoning", "encrypted_content": wrapped, "summary": "", "content": []}, + ] + ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input) + assert request_input == [ + {"type": "reasoning", "summary": "plain string thought"}, + {"type": "reasoning", "content": [{"type": "output_text", "text": "in content"}]}, + ] + + def test_leaves_input_untouched_when_no_encrypted_reasoning(self): + request_input = [ + {"role": "user", "content": "first turn"}, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "no blob here"}]}, + {"role": "user", "content": "second turn"}, + ] + before = [dict(item) for item in request_input] + ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input) + assert request_input == before + + +def _cross_group_request_kwargs(): + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a") + return { + "litellm_metadata": {}, + "input": [ + {"role": "user", "content": "ZEBRA: why is the sky blue?"}, + { + "type": "reasoning", + "encrypted_content": wrapped, + "summary": [{"type": "summary_text", "text": "scattering"}], + }, + {"type": "message", "role": "assistant", "content": "Rayleigh scattering."}, + {"role": "user", "content": "KIWI: and sunsets?"}, + ], + } + + +@pytest.mark.asyncio +async def test_affinity_strips_encrypted_reasoning_when_routed_to_another_model_group(): + """ + An auto-router tier change (or a model switch with no boundary peer): the + routed pool holds no deployment of the origin's model group. The origin is + healthy, so a 503 would be wrong; the follow-up dispatches to the routed + pool with the origin's encrypted reasoning stripped and its summary kept. + """ + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + originating = _make_originating_mock(None, "key-a", model_name="gpt-reasoning-tier") + mock_router = _make_router_mock_with_cooldown( + originating, cooldown_entries=[], routed_group_model_ids=["deployment-b"] + ) + check = EncryptedContentAffinityCheck(router=mock_router) + routed_pool = [ + { + "model_info": {"id": "deployment-b"}, + "model_name": "gpt-simple-tier", + "litellm_params": { + "api_base": "https://gateway.example/v1", + "api_key": "key-b", + "model": "openai/gpt-5-nano", + }, + } + ] + request_kwargs = _cross_group_request_kwargs() + original_input = request_kwargs["input"] + + result = await check.async_filter_deployments( + model="gpt-simple-tier", + healthy_deployments=routed_pool, + messages=None, + request_kwargs=request_kwargs, + ) + + assert result is routed_pool + assert "_encrypted_content_affinity_pinned" not in request_kwargs + assert request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True + assert request_kwargs["input"] is original_input + assert [item.get("type") or item["role"] for item in original_input] == [ + "user", + "reasoning", + "message", + "user", + ] + assert original_input[1] == { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "scattering"}], + } + assert not any(isinstance(item, dict) and item.get("encrypted_content") for item in original_input) + + +@pytest.mark.asyncio +async def test_affinity_fails_fast_within_the_origins_own_group(): + """ + Negative class for the tier-change discriminator: the routed group IS the + origin's group (a same-group cooldown, not a tier change), so even with a + healthy non-origin sibling that cannot decrypt the content, the request + still fails fast and the encrypted reasoning is left intact rather than + stripped. Preserves the LIT-3051 cooldown contract. + """ + from litellm.exceptions import ServiceUnavailableError + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + originating = _make_originating_mock( + "https://account-a.openai.azure.com/", "key-a", model_name="gpt-reasoning-tier" + ) + mock_router = _make_router_mock_with_cooldown( + originating, cooldown_entries=[], routed_group_model_ids=["deployment-a", "deployment-a2"] + ) + check = EncryptedContentAffinityCheck(router=mock_router) + sibling_pool = [ + { + "model_info": {"id": "deployment-a2"}, + "model_name": "gpt-reasoning-tier", + "litellm_params": { + "api_base": "https://account-a2.openai.azure.com/", + "api_key": "key-a2", + "model": "azure/gpt-5.4", + }, + } + ] + request_kwargs = _cross_group_request_kwargs() + + with pytest.raises(ServiceUnavailableError): + await check.async_filter_deployments( + model="gpt-reasoning-tier", + healthy_deployments=sibling_pool, + messages=None, + request_kwargs=request_kwargs, + ) + + assert request_kwargs["input"][1].get("encrypted_content") + + +@pytest.mark.asyncio +async def test_affinity_does_not_strip_when_group_is_spelled_differently_but_same_by_id(): + """ + The discriminator must key on deployment-id membership, not on the model-group + name string. Here the origin's configured group is spelled ``openai/gpt-5.4-mini`` + while the routed group is the canonical ``gpt-5.4-mini``: same group, different + spelling. A name compare (``originating.model_name != model``) would read this as + a tier change and strip the reasoning it did not have to. Because the origin's id + is a member of the routed group, this is a same-group cooldown instead: the request + fails fast and the encrypted reasoning is left intact. + """ + from litellm.exceptions import ServiceUnavailableError + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + originating = _make_originating_mock(None, "key-a", model_name="openai/gpt-5.4-mini") + mock_router = _make_router_mock_with_cooldown( + originating, cooldown_entries=[], routed_group_model_ids=["deployment-mini-a", "deployment-mini-b"] + ) + check = EncryptedContentAffinityCheck(router=mock_router) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-mini-a") + sibling_pool = [ + { + "model_info": {"id": "deployment-mini-b"}, + "model_name": "gpt-5.4-mini", + "litellm_params": { + "api_base": "https://gateway.example/v1", + "api_key": "key-b", + "model": "openai/gpt-5.4-mini", + }, + } + ] + request_kwargs = { + "litellm_metadata": {}, + "input": [ + {"role": "user", "content": "why is the sky blue?"}, + { + "type": "reasoning", + "encrypted_content": wrapped, + "summary": [{"type": "summary_text", "text": "scattering"}], + }, + {"role": "user", "content": "and sunsets?"}, + ], + } + + with pytest.raises(ServiceUnavailableError): + await check.async_filter_deployments( + model="gpt-5.4-mini", + healthy_deployments=sibling_pool, + messages=None, + request_kwargs=request_kwargs, + ) + + assert request_kwargs["input"][1].get("encrypted_content") + + +@pytest.mark.asyncio +async def test_affinity_honors_router_candidate_ids_for_team_and_pattern_routes(): + """ + The exact `model_name` index does not include team-public or pattern routes, so a + same-group cooldown reached only through one of those would be misread as a tier change + and stripped. The check asks the router for the candidate ids it resolves for the route + (`get_candidate_model_ids_for_route`), which covers those paths, rather than the bare + index. Here that set marks the origin as a candidate, so the request fails fast with its + reasoning intact, and the routed group and team are passed through to the router. + """ + from litellm.exceptions import ServiceUnavailableError + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + originating = _make_originating_mock(None, "key-a", model_name="model_name_teamA_uuid") + mock_router = _make_router_mock_with_cooldown( + originating, cooldown_entries=[], routed_group_model_ids=["deployment-team-a", "deployment-team-b"] + ) + check = EncryptedContentAffinityCheck(router=mock_router) + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-team-a") + sibling_pool = [ + { + "model_info": {"id": "deployment-team-b"}, + "model_name": "team-public-model", + "litellm_params": { + "api_base": "https://gateway.example/v1", + "api_key": "key-b", + "model": "openai/gpt-5.4-mini", + }, + } + ] + request_kwargs = { + "litellm_metadata": {"user_api_key_team_id": "teamA"}, + "input": [ + {"role": "user", "content": "why is the sky blue?"}, + { + "type": "reasoning", + "encrypted_content": wrapped, + "summary": [{"type": "summary_text", "text": "scattering"}], + }, + {"role": "user", "content": "and sunsets?"}, + ], + } + + with pytest.raises(ServiceUnavailableError): + await check.async_filter_deployments( + model="team-public-model", + healthy_deployments=sibling_pool, + messages=None, + request_kwargs=request_kwargs, + ) + + assert request_kwargs["input"][1].get("encrypted_content") + mock_router.get_candidate_model_ids_for_route.assert_called_once_with(model="team-public-model", team_id="teamA") diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index bc79c5f6589..80da922724d 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -14728,3 +14728,45 @@ def test_router_stays_quiet_when_a_deployment_drop_params_is_a_flag(value, caplo ) assert "is not a flag value" not in caplog.text + + +def test_get_candidate_model_ids_for_route_covers_model_name_and_pattern(): + """ + get_candidate_model_ids_for_route resolves a route the way the router does, so a + pre-call check can tell a genuine cross-group route from same-group unavailability. + A concrete model group returns its member ids; a wildcard/pattern deployment is + included for a concrete model it matches, which the bare model_name index misses. + Regression guard for the LIT-7195 tier-change discriminator's team/pattern gaps. + """ + router = Router( + model_list=[ + { + "model_name": "grp", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-a", "api_base": "https://x.invalid"}, + "model_info": {"id": "dep-a"}, + }, + { + "model_name": "grp", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-b", "api_base": "https://x.invalid"}, + "model_info": {"id": "dep-b"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-c", "api_base": "https://x.invalid"}, + "model_info": {"id": "dep-wild"}, + }, + ] + ) + + assert router.get_candidate_model_ids_for_route(model="grp") == frozenset({"dep-a", "dep-b"}) + assert "dep-wild" in router.get_candidate_model_ids_for_route(model="openai/gpt-4o-some-new-model") + + +def test_deployment_ids_stringifies_ids_and_skips_entries_without_a_model_info_id(): + deployments = ( + {"model_info": {"id": "a"}}, + {"model_info": {"id": 2}}, + {"model_info": {}}, + {"no_model_info": True}, + ) + assert Router._deployment_ids(deployments) == frozenset({"a", "2"}) From 096984bfc2f146e6e4bb6c259e8daab885a5e5f6 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:19:29 -0700 Subject: [PATCH 201/411] fix(proxy): pin multi-root CA bundle to the server's root before handing it to Prisma (#40428) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/db/db_url_settings.py | 107 +++++++++- .../proxy/db/test_db_url_settings.py | 199 ++++++++++++++++-- 2 files changed, 282 insertions(+), 24 deletions(-) diff --git a/litellm/proxy/db/db_url_settings.py b/litellm/proxy/db/db_url_settings.py index 4a0231ad9df..d1e4b3e92b8 100644 --- a/litellm/proxy/db/db_url_settings.py +++ b/litellm/proxy/db/db_url_settings.py @@ -34,12 +34,20 @@ writer's connection params (pool size, timeouts, pgbouncer mode) for the ones the reader URL does not pin itself. """ +import _ssl +import hashlib import os +import socket +import ssl +import struct +import sys +import tempfile import urllib.parse -from collections.abc import Mapping +from collections.abc import Callable, Mapping, Sequence from functools import partial +from pathlib import Path from types import MappingProxyType -from typing import Annotated, Final, cast +from typing import Annotated, Final, Protocol, TypeAlias, cast from pydantic import AliasChoices, BeforeValidator, Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -126,21 +134,100 @@ def add_missing_query_params(url: str, params: Mapping[str, str | int | float]) LIBPQ_VERIFY_SSLMODES: Final[frozenset[str]] = frozenset({"verify-ca", "verify-full"}) +PEM_CERT_HEADER: Final = b"-----BEGIN CERTIFICATE-----" +PG_SSL_REQUEST: Final = struct.pack("!ii", 8, 80877103) +TLS_PROBE_TIMEOUT_SECONDS: Final = 10.0 + +RootCertResolver: TypeAlias = Callable[[str, str, int], str] # mutable-ok: Callable parameter syntax -def translate_libpq_ssl_params(url: str) -> str: +class _VerifiedChainSource(Protocol): + def get_verified_chain(self) -> Sequence[_ssl.Certificate] | None: ... + + +def _verified_chain_der(tls: ssl.SSLSocket) -> tuple[bytes, ...]: + if sys.version_info >= (3, 13): + return tuple(tls.get_verified_chain()) + legacy: Final = cast( # cast-ok: the stub omits _sslobj, the C object has get_verified_chain since 3.10 + "_VerifiedChainSource | None", + tls._sslobj, # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # public API only from 3.13 + ) + chain: Final = () if legacy is None else legacy.get_verified_chain() or () + return tuple(cert.public_bytes(_ssl.ENCODING_DER) for cert in chain) + + +def _server_trust_anchor(cafile: str, host: str, port: int) -> bytes | None: + try: + context: Final = ssl.create_default_context(cafile=cafile) + with socket.create_connection((host, port), timeout=TLS_PROBE_TIMEOUT_SECONDS) as raw: + raw.sendall(PG_SSL_REQUEST) + if raw.recv(1) != b"S": + return None + with context.wrap_socket(raw, server_hostname=host) as tls: + chain: Final = _verified_chain_der(tls) + except (OSError, ValueError): + return None + return chain[-1] if chain else None + + +def pin_bundle_root(cert_path: str, host: str, port: int) -> str: + """Reduce a multi-root CA bundle to the one root that verifies ``host``. + + Prisma's ``sslcert`` loads a single PEM certificate (native-tls + ``Certificate::from_pem``), so pointing it at a bundle such as the AWS RDS + global bundle trusts only the first of its 108 regional roots and the + handshake fails with "unable to get local issuer certificate" for every + other region. A single-certificate file is returned as is. For a bundle, + one verifying handshake (chain and hostname, whole bundle as trust store) + identifies the trust anchor the server actually chains to, which is + written to a single-certificate file for Prisma. If the probe fails the + bundle path is returned unchanged, so Prisma fails closed exactly as + before rather than trusting anything the bundle would not. + """ + try: + if Path(cert_path).read_bytes().count(PEM_CERT_HEADER) < 2: + return cert_path + except OSError: + return cert_path + root: Final = _server_trust_anchor(cert_path, host, port) + if root is None: + return cert_path + pinned: Final = Path(tempfile.gettempdir()) / f"litellm-sslcert-{hashlib.sha256(root).hexdigest()[:16]}.pem" + return str(pinned) if _replace_file(pinned, ssl.DER_cert_to_PEM_cert(root)) else cert_path + + +def _replace_file(target: Path, content: str) -> bool: + """Write ``content`` to a private temp file and rename it over ``target``, so + readers never see a partial file and a symlink planted at ``target`` is + replaced rather than followed.""" + try: + fd, staged = tempfile.mkstemp(dir=target.parent, prefix=f"{target.name}.") + except OSError: + return False + try: + with os.fdopen(fd, "w") as handle: + handle.write(content) + os.replace(staged, target) + except OSError: + Path(staged).unlink(missing_ok=True) + return False + return True + + +def translate_libpq_ssl_params(url: str, resolve_root_cert: RootCertResolver = pin_bundle_root) -> str: """Rewrite libpq's certificate-verification params into Prisma's dialect. Prisma's engine only knows ``sslmode=disable|prefer|require``, ``sslcert`` - (the CA bundle) and ``sslaccept=strict``. It silently discards + (a single CA certificate) and ``sslaccept=strict``. It silently discards ``sslrootcert`` and downgrades ``sslmode=verify-ca`` / ``verify-full`` to ``prefer``, so a URL copied from libpq / RDS docs connects over TLS with no certificate check at all. ``verify-ca`` and ``verify-full`` both become ``require`` (Prisma has no CA-only mode), ``sslrootcert`` becomes - ``sslcert``, and either one turns on ``sslaccept=strict`` (chain and - hostname), matching libpq where a root cert makes ``require`` verify. - Prisma params the operator pinned themselves win; anything else is left - untouched. + ``sslcert`` (run through ``resolve_root_cert``, which pins a multi-root + bundle down to the server's root), and either one turns on + ``sslaccept=strict`` (chain and hostname), matching libpq where a root + cert makes ``require`` verify. Prisma params the operator pinned + themselves win; anything else is left untouched. """ parsed: Final = urllib.parse.urlsplit(url) pairs: Final = tuple(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)) @@ -154,7 +241,9 @@ def translate_libpq_ssl_params(url: str) -> str: if key != "sslrootcert" ) root_cert: Final = tuple( - ("sslcert", value) for key, value in pairs if key == "sslrootcert" and "sslcert" not in keys + ("sslcert", resolve_root_cert(value, parsed.hostname or "", parsed.port or int(DEFAULT_POSTGRES_PORT))) + for key, value in pairs + if key == "sslrootcert" and "sslcert" not in keys ) strict: Final = () if "sslaccept" in keys else (("sslaccept", "strict"),) query: Final = urllib.parse.urlencode(translated + root_cert + strict) diff --git a/tests/test_litellm/proxy/db/test_db_url_settings.py b/tests/test_litellm/proxy/db/test_db_url_settings.py index ba342342366..875dca4bee3 100644 --- a/tests/test_litellm/proxy/db/test_db_url_settings.py +++ b/tests/test_litellm/proxy/db/test_db_url_settings.py @@ -11,15 +11,30 @@ clobber a pre-existing ``DATABASE_URL_READ_REPLICA``. A pre-existing ``DATABASE_URL`` (password auth) is likewise left untouched. """ +import datetime +import hashlib import os +import socket +import ssl +import tempfile +import threading import urllib.parse +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final from unittest.mock import patch import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec from pydantic import ValidationError from litellm.proxy.db.db_url_settings import ( + PG_SSL_REQUEST, DatabaseURLSettings, + translate_libpq_ssl_params, unsupported_db_scheme, unsupported_db_scheme_message, ) @@ -381,9 +396,7 @@ def test_writer_password_is_percent_encoded(monkeypatch): def test_writer_url_not_clobbered_when_already_set(monkeypatch): """An operator-pinned DATABASE_URL (e.g. helm's $(VAR) assembly) always wins over the discrete fields.""" - monkeypatch.setenv( - "DATABASE_URL", "postgresql://pinned:url@db.example.com:5432/litellm_db" - ) + monkeypatch.setenv("DATABASE_URL", "postgresql://pinned:url@db.example.com:5432/litellm_db") monkeypatch.setenv("DATABASE_HOST", "writer.example.com") monkeypatch.setenv("DATABASE_USER", "litellm") monkeypatch.setenv("DATABASE_NAME", "litellm_db") @@ -515,9 +528,7 @@ def test_apply_to_env_rejects_pinned_sqlite_direct_url(monkeypatch): def test_apply_to_env_rejects_pinned_non_postgres_reader(monkeypatch): monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@writer.example.com:5432/db") - monkeypatch.setenv( - "DATABASE_URL_READ_REPLICA", "mysql://u:p@reader.example.com:3306/db" - ) + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", "mysql://u:p@reader.example.com:3306/db") with pytest.raises(RuntimeError, match=r"DATABASE_URL_READ_REPLICA.*mysql"): _apply() @@ -542,15 +553,11 @@ def test_reader_inherits_writer_connection_params(monkeypatch): "DATABASE_URL", "postgresql://u:p@writer.example.com:5432/db?connection_limit=3&pool_timeout=20&pgbouncer=true", ) - monkeypatch.setenv( - "DATABASE_URL_READ_REPLICA", "postgresql://u:p@reader.example.com:5432/db" - ) + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", "postgresql://u:p@reader.example.com:5432/db") _apply() - query = urllib.parse.parse_qs( - urllib.parse.urlsplit(os.environ["DATABASE_URL_READ_REPLICA"]).query - ) + query = urllib.parse.parse_qs(urllib.parse.urlsplit(os.environ["DATABASE_URL_READ_REPLICA"]).query) assert query["connection_limit"] == ["3"] assert query["pool_timeout"] == ["20"] assert query["pgbouncer"] == ["true"] @@ -568,9 +575,7 @@ def test_reader_keeps_its_own_pinned_connection_params(monkeypatch): _apply() - query = urllib.parse.parse_qs( - urllib.parse.urlsplit(os.environ["DATABASE_URL_READ_REPLICA"]).query - ) + query = urllib.parse.parse_qs(urllib.parse.urlsplit(os.environ["DATABASE_URL_READ_REPLICA"]).query) assert query["connection_limit"] == ["50"] assert query["pool_timeout"] == ["20"] @@ -776,6 +781,170 @@ def test_libpq_verify_full_and_sslrootcert_become_prisma_strict_sslcert(monkeypa } +def _issue_cert( + subject: str, issuer: x509.Certificate | None, issuer_key: ec.EllipticCurvePrivateKey | None, ca: bool +) -> tuple[x509.Certificate, ec.EllipticCurvePrivateKey]: + key: Final = ec.generate_private_key(ec.SECP256R1()) + name: Final = x509.Name((x509.NameAttribute(x509.NameOID.COMMON_NAME, subject),)) + now: Final = datetime.datetime.now(datetime.timezone.utc) + builder: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(issuer.subject if issuer else name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(minutes=5)) + .not_valid_after(now + datetime.timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=ca, path_length=None), critical=True) + .add_extension(x509.SubjectAlternativeName((x509.DNSName("localhost"),)), critical=False) + ) + return builder.sign(issuer_key or key, hashes.SHA256()), key + + +def _pem(cert: x509.Certificate) -> bytes: + return cert.public_bytes(serialization.Encoding.PEM) + + +class _TlsPostgresStub: + """Answers one libpq ``SSLRequest`` with ``S`` and serves ``leaf + intermediate``.""" + + def __init__(self, chain_pem: Path, key_pem: Path) -> None: + self.context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.context.load_cert_chain(str(chain_pem), str(key_pem)) + self.listener: Final = socket.create_server(("127.0.0.1", 0)) + self.port: Final[int] = self.listener.getsockname()[1] + self.thread: Final = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + def _serve(self) -> None: + with self.listener: + while True: + try: + conn: socket.socket = self.listener.accept()[0] + except OSError: + return + with conn: + try: + if conn.recv(8) == PG_SSL_REQUEST: + conn.sendall(b"S") + with self.context.wrap_socket(conn, server_side=True) as tls: + tls.recv(1) + except OSError: + continue + + +@dataclass(frozen=True, slots=True) +class _RdsLikePki: + bundle: Path + wrong_bundle: Path + root: Path + port: int + + +@pytest.fixture +def rds_like_pki(tmp_path: Path) -> Iterator[_RdsLikePki]: + """An RDS-shaped trust setup: the server sends leaf + intermediate, the + bundle holds only self-signed roots, and the right root is not first.""" + root, root_key = _issue_cert("Real Root CA", None, None, ca=True) + decoys: Final = tuple(_issue_cert(f"Decoy Root CA {i}", None, None, ca=True)[0] for i in range(3)) + intermediate, intermediate_key = _issue_cert("Intermediate CA", root, root_key, ca=True) + leaf, leaf_key = _issue_cert("localhost", intermediate, intermediate_key, ca=False) + chain_pem: Final = tmp_path / "server-chain.pem" + chain_pem.write_bytes(_pem(leaf) + _pem(intermediate)) + key_pem: Final = tmp_path / "server.key" + key_pem.write_bytes( + leaf_key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ) + ) + bundle: Final = tmp_path / "global-bundle.pem" + bundle.write_bytes(b"".join(_pem(decoy) for decoy in decoys) + _pem(root)) + wrong_bundle: Final = tmp_path / "wrong-bundle.pem" + wrong_bundle.write_bytes(b"".join(_pem(decoy) for decoy in decoys)) + root_pem: Final = tmp_path / "root.pem" + root_pem.write_bytes(_pem(root)) + stub: Final = _TlsPostgresStub(chain_pem, key_pem) + yield _RdsLikePki(bundle=bundle, wrong_bundle=wrong_bundle, root=root_pem, port=stub.port) + stub.listener.close() + + +def _params(url: str) -> tuple[tuple[str, str], ...]: + return tuple(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query, keep_blank_values=True)) + + +def test_multi_root_bundle_is_pinned_to_the_root_the_server_chains_to( + monkeypatch: pytest.MonkeyPatch, rds_like_pki: _RdsLikePki +): + """Prisma's ``sslcert`` loads only the first certificate of the file, so + handing it the whole RDS bundle trusts one region's root and fails with + "unable to get local issuer certificate" everywhere else. The URL Prisma + receives must point at a single-certificate file holding the server's root.""" + monkeypatch.setenv( + "DATABASE_URL", + f"postgresql://u:p@localhost:{rds_like_pki.port}/litellm_db?sslmode=verify-full&sslrootcert={rds_like_pki.bundle}", + ) + + _apply() + + (sslmode, sslcert, sslaccept, _) = _params(os.environ["DATABASE_URL"]) + assert (sslmode, sslaccept) == (("sslmode", "require"), ("sslaccept", "strict")) + assert sslcert[0] == "sslcert" and sslcert[1] != str(rds_like_pki.bundle) + assert Path(sslcert[1]).read_bytes() == rds_like_pki.root.read_bytes() + + +def test_pinned_root_replaces_a_planted_symlink_instead_of_writing_through_it( + monkeypatch: pytest.MonkeyPatch, rds_like_pki: _RdsLikePki, tmp_path: Path +): + """The pinned file has a predictable name in a shared temp dir, so a symlink + planted there must not redirect the write onto its target.""" + monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) + root_der: Final = x509.load_pem_x509_certificate(rds_like_pki.root.read_bytes()).public_bytes( + serialization.Encoding.DER + ) + pinned: Final = tmp_path / f"litellm-sslcert-{hashlib.sha256(root_der).hexdigest()[:16]}.pem" + victim: Final = tmp_path / "victim.txt" + victim.write_text("untouched") + pinned.symlink_to(victim) + monkeypatch.setenv( + "DATABASE_URL", + f"postgresql://u:p@localhost:{rds_like_pki.port}/litellm_db?sslmode=verify-full&sslrootcert={rds_like_pki.bundle}", + ) + + _apply() + + assert ("sslcert", str(pinned)) in _params(os.environ["DATABASE_URL"]) + assert victim.read_text() == "untouched" + assert not pinned.is_symlink() and pinned.read_bytes() == rds_like_pki.root.read_bytes() + + +def test_bundle_without_the_servers_root_is_passed_through_unchanged( + monkeypatch: pytest.MonkeyPatch, rds_like_pki: _RdsLikePki +): + """Nothing in the bundle verifies the server, so no root is pinned and + Prisma keeps rejecting the connection instead of trusting a root the + operator never shipped.""" + monkeypatch.setenv( + "DATABASE_URL", + f"postgresql://u:p@localhost:{rds_like_pki.port}/litellm_db" + f"?sslmode=verify-full&sslrootcert={rds_like_pki.wrong_bundle}", + ) + + _apply() + + assert ("sslcert", str(rds_like_pki.wrong_bundle)) in _params(os.environ["DATABASE_URL"]) + + +def test_root_cert_resolver_receives_the_urls_host_and_default_port(): + def resolver(cert_path: str, host: str, port: int) -> str: + return f"/pinned/{host}/{port}{cert_path}" + + url: Final = translate_libpq_ssl_params( + "postgresql://u:p@db.example.com/litellm_db?sslmode=verify-full&sslrootcert=/certs/bundle.pem", resolver + ) + + assert ("sslcert", "/pinned/db.example.com/5432/certs/bundle.pem") in _params(url) + + def test_libpq_verify_ca_becomes_prisma_strict(monkeypatch): monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@db.example.com:5432/litellm_db?sslmode=verify-ca") From 00e4380cb5b84b9681cbf6e14886cdbaf328cf4b Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 19:15:44 +0000 Subject: [PATCH 202/411] fix(registry): add Bedrock gpt-6-astra CRIS + mantle profiles, embed-v4/pegasus global profiles, gpt-image-2.5 entries; cap Vertex grok-4.1-fast output at 128k Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 188 +++++++++++++++++- model_prices_and_context_window.json | 188 +++++++++++++++++- 2 files changed, 368 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cdd9fc7b6d4..664c2737157 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -759,6 +759,14 @@ "mode": "chat", "supports_video_input": true }, + "global.twelvelabs.pegasus-1-2-v1:0": { + "input_cost_per_video_per_second": 0.00049, + "output_cost_per_token": 7.5e-06, + "litellm_provider": "bedrock", + "mode": "chat", + "supports_video_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "amazon.titan-text-express-v1": { "input_cost_per_token": 1.3e-06, "litellm_provider": "bedrock", @@ -14431,6 +14439,28 @@ "output_vector_size": 1536, "supports_embedding_image_input": true }, + "us.cohere.embed-v4:0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "supports_embedding_image_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "global.cohere.embed-v4:0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "supports_embedding_image_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "cohere/embed-v4.0": { "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", @@ -29748,6 +29778,66 @@ "supports_vision": true, "supports_pdf_input": true }, + "gpt-image-2.5-flare": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-flare-2026-09-08": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-sunburst": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-sunburst-2026-09-08": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, "low/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, @@ -48040,8 +48130,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 128000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://docs.x.ai/developers/models", @@ -48056,8 +48146,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 128000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://docs.x.ai/developers/models", @@ -55429,6 +55519,96 @@ "supports_reasoning": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-6-astra": { + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, + "us.openai.gpt-6-astra": { + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, + "global.openai.gpt-6-astra": { + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "output_cost_per_token": 5e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cdd9fc7b6d4..664c2737157 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -759,6 +759,14 @@ "mode": "chat", "supports_video_input": true }, + "global.twelvelabs.pegasus-1-2-v1:0": { + "input_cost_per_video_per_second": 0.00049, + "output_cost_per_token": 7.5e-06, + "litellm_provider": "bedrock", + "mode": "chat", + "supports_video_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "amazon.titan-text-express-v1": { "input_cost_per_token": 1.3e-06, "litellm_provider": "bedrock", @@ -14431,6 +14439,28 @@ "output_vector_size": 1536, "supports_embedding_image_input": true }, + "us.cohere.embed-v4:0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "supports_embedding_image_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "global.cohere.embed-v4:0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "supports_embedding_image_input": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "cohere/embed-v4.0": { "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", @@ -29748,6 +29778,66 @@ "supports_vision": true, "supports_pdf_input": true }, + "gpt-image-2.5-flare": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-flare-2026-09-08": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-sunburst": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, + "gpt-image-2.5-sunburst-2026-09-08": { + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_vision": true, + "supports_pdf_input": true, + "source": "https://developers.openai.com/api/docs/pricing" + }, "low/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, @@ -48040,8 +48130,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 128000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://docs.x.ai/developers/models", @@ -48056,8 +48146,8 @@ "input_cost_per_token": 2e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 128000, - "max_output_tokens": 2000000, - "max_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://docs.x.ai/developers/models", @@ -55429,6 +55519,96 @@ "supports_reasoning": true, "supports_vision": true }, + "bedrock_mantle/openai.gpt-6-astra": { + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, + "us.openai.gpt-6-astra": { + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, + "global.openai.gpt-6-astra": { + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "output_cost_per_token": 5e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, From 34d2d010c3f0f08ab9c13700c7004d98377309fa Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 19:33:59 +0000 Subject: [PATCH 203/411] test(cli): drop lite e2e tests, the e2e runner does not install the package Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/coverage_registry/other.yaml | 2 - tests/e2e/other/test_cli_cost_map_e2e.py | 104 ----------------------- 2 files changed, 106 deletions(-) delete mode 100644 tests/e2e/other/test_cli_cost_map_e2e.py diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index a7ba0dbb8cb..814ebae2e0b 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -48,5 +48,3 @@ - {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"} - {id: other.a2a.version.serves_pinned_0_3, module: other, tier: P1, area: a2a, assertions: [serves_pinned_0_3], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 0.3 returns the flat 0.3 message shape (parts on the result)"} - {id: other.a2a.version.serves_pinned_1_0, module: other, tier: P1, area: a2a, assertions: [serves_pinned_1_0], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 1.0 returns the nested 1.0 message shape (result.message with ROLE_AGENT)"} -- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} -- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} diff --git a/tests/e2e/other/test_cli_cost_map_e2e.py b/tests/e2e/other/test_cli_cost_map_e2e.py deleted file mode 100644 index 62cd3d6c3f5..00000000000 --- a/tests/e2e/other/test_cli_cost_map_e2e.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -import os -import shutil -import subprocess -import threading -from collections.abc import Mapping -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Final - -import pytest -from e2e_config import MASTER_KEY, PROXY_BASE_URL -from proxy_client import ProxyClient - -pytestmark = pytest.mark.e2e - - -def _start_cost_map_server(request_log: Path) -> tuple[ThreadingHTTPServer, threading.Thread]: - class CostMapHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - with request_log.open("a", encoding="utf-8") as log_file: - log_file.write(f"{self.path}\n") - body: Final = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: object) -> None: - return - - server: Final = ThreadingHTTPServer(("127.0.0.1", 0), CostMapHandler) - thread: Final = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - return server, thread - - -def _lite_env(server: ThreadingHTTPServer, api_key: str | None) -> dict[str, str]: - base_env: Final = {key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"} - return { - **base_env, - "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", - "LITELLM_PROXY_URL": PROXY_BASE_URL, - **({"LITELLM_PROXY_API_KEY": api_key} if api_key is not None else {}), - } - - -def _run_lite( - args: tuple[str, ...], - server: ThreadingHTTPServer, - env: Mapping[str, str], -) -> subprocess.CompletedProcess[str]: - lite_path: Final = shutil.which("lite") - assert lite_path is not None, "the installed lite executable is required for e2e coverage" - try: - return subprocess.run( - [lite_path, *args], - capture_output=True, - text=True, - timeout=60, - env=env, - ) - finally: - server.shutdown() - server.server_close() - - -def _request_count(request_log: Path) -> int: - return request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 - - -class TestLiteCliCostMapFetch: - @pytest.mark.covers("other.cli.model_cost_map.version_skips_fetch") - def test_lite_version_makes_no_cost_map_request(self, tmp_path: Path) -> None: - request_log: Final = tmp_path / "requests.log" - server, thread = _start_cost_map_server(request_log) - env: Final = _lite_env(server, None) - try: - result: Final = _run_lite(("--version",), server, env) - finally: - thread.join(timeout=10) - - assert result.returncode == 0 - assert "LiteLLM Proxy CLI Version" in result.stdout - assert _request_count(request_log) == 0 - - @pytest.mark.covers("other.cli.model_cost_map.models_list_skips_fetch") - def test_lite_models_list_uses_proxy_not_cost_map(self, tmp_path: Path, proxy: ProxyClient) -> None: - model_names: Final = tuple(entry.model_name for entry in proxy.model_info()) - assert model_names - request_log: Final = tmp_path / "requests.log" - server, thread = _start_cost_map_server(request_log) - env: Final = _lite_env(server, MASTER_KEY) - try: - result: Final = _run_lite(("models", "list"), server, env) - finally: - thread.join(timeout=10) - - assert result.returncode == 0 - assert result.stdout.strip() - assert any(model_name in result.stdout for model_name in model_names) - assert _request_count(request_log) == 0 From 8bb6d8c120e2c116a719495ecf2401a2bab9dd5a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:34:08 -0700 Subject: [PATCH 204/411] chore(lint): bring ANN202 and BLE001 back under the strict-rule budget Annotate the return types of dispatch_async and transform_then_dispatch in llm_http_handler and _send_batch in azure_sentinel, and mark four legitimate broad catches with the repo's noqa convention, so the promote PR's lint job passes the strict gate again. Supersedes #40328. --- litellm/integrations/azure_sentinel/azure_sentinel.py | 4 +++- litellm/litellm_core_utils/get_model_cost_map.py | 2 +- litellm/litellm_core_utils/llm_cost_calc/utils.py | 2 +- litellm/llms/custom_httpx/llm_http_handler.py | 4 ++-- litellm/proxy/_experimental/mcp_server/server.py | 2 +- litellm/proxy/management_endpoints/cost_tracking_settings.py | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/azure_sentinel/azure_sentinel.py b/litellm/integrations/azure_sentinel/azure_sentinel.py index eba6c862f7a..db5f790615f 100644 --- a/litellm/integrations/azure_sentinel/azure_sentinel.py +++ b/litellm/integrations/azure_sentinel/azure_sentinel.py @@ -21,6 +21,8 @@ from types import MappingProxyType from typing import Final, TypeVar from urllib.parse import urlparse +import httpx + from litellm._logging import verbose_logger from litellm.integrations.batch_utils import ( BatchSendCancelled, @@ -418,7 +420,7 @@ class AzureSentinelLogger(CustomBatchLogger): "Content-Type": "application/json", } - async def _send_batch(batch: Sequence[_QueuedPayload]): + async def _send_batch(batch: Sequence[_QueuedPayload]) -> httpx.Response: body: Final = safe_dumps(batch) return await self.async_httpx_client.post( url=api_endpoint, diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index f81ddbfee2e..37375f25e46 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -585,7 +585,7 @@ def _retry_remote_fetch_in_background( _cost_map_source_info.fallback_reason = None _cost_map_source_info.loaded_at = datetime.now(timezone.utc) adopt_model_cost_map(finalized) - except Exception as e: + except Exception as e: # noqa: BLE001 # a failed background retry must not kill the task; the backup stays verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 5675c59733d..e5977ca4156 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1444,7 +1444,7 @@ def get_billed_token_rates( return _custom_pricing_rates(custom_cost_per_token) try: model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) - except Exception: + except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model: no rates return None return _cost_map_billed_rates( model_info=model_info, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 7587b963a38..84b03e1955a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -579,7 +579,7 @@ class BaseLLMHTTPHandler: data: dict[str, object], # mutable-ok: async_completion takes dict signed_headers: dict[str, object], # mutable-ok: async_completion takes dict signed_json_body: bytes | None, - ): + ) -> Coroutine[object, object, ModelResponse | CustomStreamWrapper]: async_client: Final = client if isinstance(client, AsyncHTTPHandler) else None if stream is True: return self.acompletion_stream_function( @@ -626,7 +626,7 @@ class BaseLLMHTTPHandler: if acompletion is True and provider_config.uses_async_transform_request: - async def transform_then_dispatch(): + async def transform_then_dispatch() -> ModelResponse | CustomStreamWrapper: transformed: Final = cast( # cast-ok: async_transform_request is declared as a bare dict "dict[str, object]", await provider_config.async_transform_request( diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 9a52da0cab1..f8f1c563811 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -1053,7 +1053,7 @@ if MCP_AVAILABLE: route="/mcp/call_tool", traceback_str=failure_traceback, ) - except Exception: + except Exception: # noqa: BLE001 # a failing failure hook must not mask the tool call's own error verbose_logger.exception("Error logging failed MCP proxy tool call") raise if proxy_logging_obj is not None: diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 493f75008c3..dc0da63555f 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -643,7 +643,7 @@ async def estimate_cost( custom_cost_per_token=resolved.custom_cost_per_token, litellm_logging_obj=litellm_logging_obj, ) - except Exception as e: + except Exception as e: # noqa: BLE001 # completion_cost raises a bare Exception for an unpriceable model raise HTTPException( status_code=404, detail={ From 35def27e7b145c7aef7463bef34db4e56a2e8d9b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:37:29 -0700 Subject: [PATCH 205/411] fix(convert_dict_to_response): accept only a real list as choices and keep /v1/messages alive on an empty one Narrows the no-choices guard so a dict, string, or None still raises the APIError while an empty list passes through, guards the non-stream Anthropic bridge against indexing an empty choices list, and repairs test_completion_missing_role, whose raw-response mock was patched in as the create() callable itself so the handler only ever saw a MagicMock --- .../convert_dict_to_response.py | 8 ++--- .../adapters/transformation.py | 3 +- .../test_convert_dict_to_response.py | 30 +++++++++++++++++++ ...al_pass_through_adapters_transformation.py | 15 ++++++++++ tests/test_litellm/test_main.py | 2 +- 5 files changed, 52 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 7d8072ba622..29b75812bfb 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -3,7 +3,7 @@ import json import re import time import traceback -from collections.abc import Iterable, Sequence +from collections.abc import Sequence from typing import Final, Literal, cast import litellm @@ -179,7 +179,7 @@ async def convert_to_streaming_response_async( choice_list: Final[list[StreamingChoices]] = [] - if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): + if not isinstance(response_object.get("choices"), list): from litellm.exceptions import APIError raise APIError( @@ -287,7 +287,7 @@ def convert_to_streaming_response( model_response_object: Final = ModelResponseStream() choice_list: Final[list[StreamingChoices]] = [] - if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): + if not isinstance(response_object.get("choices"), list): from litellm.exceptions import APIError raise APIError( @@ -623,7 +623,7 @@ def convert_to_model_response_object( return convert_to_streaming_response(response_object=response_object) choice_list: Final[list[Choices]] = [] - if "choices" not in response_object or not isinstance(response_object["choices"], Iterable): + if not isinstance(response_object.get("choices"), list): from litellm.exceptions import APIError raise APIError( diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index db890662132..4bce760e943 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1487,8 +1487,9 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_content.insert(0, polyfill_result.compaction_block) ## extract finish reason + openai_finish_reason: Final = response.choices[0].finish_reason if response.choices else "stop" translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic( - openai_finish_reason=response.choices[0].finish_reason + openai_finish_reason=openai_finish_reason ) anthropic_finish_reason: Final = ( "refusal" diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 5406df691aa..ea50bff462e 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1,3 +1,5 @@ +from typing import Final + import pytest from litellm.constants import RESPONSE_FORMAT_TOOL_NAME @@ -167,3 +169,31 @@ def test_convert_missing_choices_raises_api_error() -> None: ) assert "no 'choices'" in str(exc_info.value) + +@pytest.mark.parametrize("choices", [{}, "", None, 0]) +@pytest.mark.asyncio +async def test_convert_non_list_choices_raises_api_error(choices: object) -> None: + from litellm.exceptions import APIError + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response, + convert_to_streaming_response_async, + ) + + resp: Final = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + "choices": choices, + } + with pytest.raises(APIError, match="no 'choices'"): + convert_to_model_response_object( + response_object=resp, + model_response_object=ModelResponse(), + response_type="completion", + ) + with pytest.raises(APIError, match="no 'choices'"): + list(convert_to_streaming_response(response_object=resp)) + with pytest.raises(APIError, match="no 'choices'"): + async for _ in convert_to_streaming_response_async(response_object=resp): + pass diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index c59ec70b015..00b3a0633f2 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -41,6 +41,21 @@ from litellm.types.utils import ( ) +def test_translate_openai_response_to_anthropic_empty_choices() -> None: + response: Final = ModelResponse( + id="chatcmpl-empty", + model="gemini-3.5-flash", + choices=[], + usage=Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10), + ) + + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert result["content"] == [] + assert result["stop_reason"] == "end_turn" + assert result["usage"]["input_tokens"] == 10 + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 038df3656fe..de6322f27ee 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -120,7 +120,7 @@ def test_completion_missing_role(openai_api_response): print(f"openai_api_response: {openai_api_response}") with patch.object( - client.chat.completions.with_raw_response, "create", mock_raw_response + client.chat.completions.with_raw_response, "create", MagicMock(return_value=mock_raw_response) ) as mock_create: litellm.completion( model="gpt-4o-mini", From 7b43977460249021a89ddfb285c83ce361ff90b1 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 9 Sep 2026 12:37:36 -0700 Subject: [PATCH 206/411] feat(bedrock): thread aws_session_tags into STS AssumeRole Operators can now set aws_session_tags on a Bedrock or SageMaker deployment and every AssumeRole call carries them as STS session tags, so trust policies gated on sts:TagSession admit the session and CloudTrail and cost reports see the tags. Tags are validated up front, sorted into the credential cache key so tag order does not fork sessions, stripped from invoke and embedding request bodies, and blocked from client request bodies like the other AWS identity params. Based on #34073. Co-authored-by: Satya Yedida <222552552+satya-rubrik@users.noreply.github.com> --- .../litellm_core_utils/get_litellm_params.py | 1 + litellm/llms/bedrock/base_aws_llm.py | 108 ++++--- litellm/llms/bedrock/batches/handler.py | 3 +- litellm/llms/bedrock/chat/converse_handler.py | 2 + litellm/llms/bedrock/common_utils.py | 2 + litellm/llms/bedrock/embed/embedding.py | 2 + litellm/llms/sagemaker/chat/handler.py | 2 + litellm/llms/sagemaker/completion/handler.py | 2 + litellm/proxy/auth/auth_utils.py | 1 + litellm/types/llms/bedrock.py | 5 + litellm/types/router.py | 4 +- ..._bedrock_dynamic_auth_params_unit_tests.py | 1 + .../llms/bedrock/batches/test_handler.py | 52 +++- .../test_amazon_moonshot_transformation.py | 1 + ...ations_anthropic_claude3_transformation.py | 4 + .../chat/test_bedrock_converse_handler.py | 54 ++++ .../bedrock/embed/test_bedrock_embedding.py | 57 ++++ .../llms/bedrock/test_base_aws_llm.py | 279 +++++++++++++++++- .../llms/bedrock/test_bedrock_common_utils.py | 56 ++++ .../sagemaker/test_sagemaker_chat_handler.py | 42 +++ .../test_sagemaker_completion_handler.py | 48 +++ .../proxy/auth/test_auth_utils.py | 4 +- tests/test_litellm/types/test_router.py | 24 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 13 + 24 files changed, 728 insertions(+), 39 deletions(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 92b32d32dc0..edd2e88f95c 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -17,6 +17,7 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset( "aws_web_identity_token", "aws_sts_endpoint", "aws_external_id", + "aws_session_tags", "aws_bedrock_runtime_endpoint", "aws_bedrock_project_id", } diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index a197702921f..ab6c4ab85d6 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -4,13 +4,14 @@ import json import os import re import urllib.parse -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from datetime import datetime from threading import Lock from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, cast, get_args, overload import httpx -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.caching.caching import DualCache @@ -26,6 +27,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.aws_partition import contains_bedrock_arn, get_aws_dns_suffix from litellm.litellm_core_utils.dd_tracing import tracer from litellm.secret_managers.main import get_secret, get_secret_str +from litellm.types.llms.bedrock import AwsSessionTag if TYPE_CHECKING: from botocore.awsrequest import AWSPreparedRequest @@ -47,6 +49,47 @@ _STS_REGION_FROM_ENDPOINT_PATTERN: Final = re.compile( SIGV4_COMPUTED_HEADERS: Final = frozenset({"authorization", "x-amz-date", "x-amz-security-token", "date"}) +_AWS_SESSION_TAGS_ADAPTER: Final[TypeAdapter[tuple[AwsSessionTag, ...]]] = TypeAdapter(tuple[AwsSessionTag, ...]) + + +def _canonical_aws_session_tags(raw_tags: object) -> tuple[AwsSessionTag, ...] | None: + if raw_tags is None: + return None + try: + validated: Final = _AWS_SESSION_TAGS_ADAPTER.validate_python(raw_tags) + except ValidationError as e: + raise ValueError( + "Invalid 'aws_session_tags' value. Expected a list of {'Key': , 'Value': } dicts, " + f"e.g. [{{'Key': 'team', 'Value': 'genai'}}]. Got: {raw_tags!r}" + ) from e + return tuple(sorted(validated, key=lambda tag: tag["Key"])) + + +class _AssumeRoleParams(TypedDict): + RoleArn: ReadOnly[str] + RoleSessionName: ReadOnly[str] + ExternalId: ReadOnly[NotRequired[str]] + Tags: ReadOnly[NotRequired[tuple[AwsSessionTag, ...]]] + + +def _assume_role_params( + aws_role_name: str, + aws_session_name: str, + aws_external_id: str | None, + aws_session_tags: Sequence[AwsSessionTag] | None, +) -> _AssumeRoleParams: + match (aws_external_id, tuple(aws_session_tags or ())): + case (None, ()): + return _AssumeRoleParams(RoleArn=aws_role_name, RoleSessionName=aws_session_name) + case (None, tags): + return _AssumeRoleParams(RoleArn=aws_role_name, RoleSessionName=aws_session_name, Tags=tags) + case (external_id, ()): + return _AssumeRoleParams(RoleArn=aws_role_name, RoleSessionName=aws_session_name, ExternalId=external_id) + case (external_id, tags): + return _AssumeRoleParams( + RoleArn=aws_role_name, RoleSessionName=aws_session_name, ExternalId=external_id, Tags=tags + ) + class BedrockRequestTarget(BaseModel): aws_region_name: str @@ -120,6 +163,7 @@ class BaseAWSLLM: "aws_sts_endpoint", "aws_bedrock_runtime_endpoint", "aws_external_id", + "aws_session_tags", ] def _get_ssl_verify(self, ssl_verify: bool | str | None = None): @@ -137,7 +181,7 @@ class BaseAWSLLM: return get_ssl_verify(ssl_verify=ssl_verify) - def get_cache_key(self, credential_args: Mapping[str, str | bool | None]) -> str: + def get_cache_key(self, credential_args: Mapping[str, str | bool | tuple[AwsSessionTag, ...] | None]) -> str: """ Generate a unique cache key based on the credential arguments. """ @@ -147,7 +191,7 @@ class BaseAWSLLM: def _get_or_set_cached_credentials( self, - credential_args: Mapping[str, str | bool | None], + credential_args: Mapping[str, str | bool | tuple[AwsSessionTag, ...] | None], credential_fetcher: Callable[[], tuple[Credentials, int | None]], ) -> Any: """ @@ -222,6 +266,7 @@ class BaseAWSLLM: aws_web_identity_token: str | None = None, aws_sts_endpoint: str | None = None, aws_external_id: str | None = None, + aws_session_tags: Sequence[AwsSessionTag] | None = None, ssl_verify: bool | str | None = None, ): """ @@ -258,6 +303,7 @@ class BaseAWSLLM: (aws_external_id, "AWS_EXTERNAL_ID"), ) ) + session_tags: Final = _canonical_aws_session_tags(aws_session_tags) verbose_logger.debug( "in get credentials\n" @@ -270,7 +316,8 @@ class BaseAWSLLM: "aws_role_name=%s\n" "aws_web_identity_token=[set=%s]\n" "aws_sts_endpoint=%s\n" - "aws_external_id=%s", + "aws_external_id=%s\n" + "aws_session_tags=%s", aws_access_key_id is not None, aws_secret_access_key is not None, aws_session_token is not None, @@ -281,6 +328,7 @@ class BaseAWSLLM: aws_web_identity_token is not None, aws_sts_endpoint, aws_external_id, + session_tags, ) args: Final = { @@ -294,6 +342,7 @@ class BaseAWSLLM: "aws_web_identity_token": aws_web_identity_token, "aws_sts_endpoint": aws_sts_endpoint, "aws_external_id": aws_external_id, + "aws_session_tags": session_tags, "ssl_verify": ssl_verify, } @@ -336,6 +385,7 @@ class BaseAWSLLM: aws_region_name=aws_region_name, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=session_tags, ssl_verify=ssl_verify, ), ) @@ -980,6 +1030,7 @@ class BaseAWSLLM: aws_sts_endpoint: str | None = None, ssl_verify: bool | str | None = None, aws_region_name: str | None = None, + aws_session_tags: Sequence[AwsSessionTag] | None = None, ) -> dict: """Handle cross-account role assumption for IRSA.""" import boto3 @@ -1032,16 +1083,9 @@ class BaseAWSLLM: # Now assume the target role verbose_logger.debug("Attempting to assume target role: %s with session: %s", aws_role_name, aws_session_name) - assume_role_params: Final = { - "RoleArn": aws_role_name, - "RoleSessionName": aws_session_name, - } - - # Add ExternalId parameter if provided - if aws_external_id is not None: - assume_role_params["ExternalId"] = aws_external_id - - return sts_client_with_creds.assume_role(**assume_role_params) + return sts_client_with_creds.assume_role( + **_assume_role_params(aws_role_name, aws_session_name, aws_external_id, aws_session_tags) + ) def _handle_irsa_same_account( self, @@ -1051,6 +1095,7 @@ class BaseAWSLLM: aws_sts_endpoint: str | None = None, ssl_verify: bool | str | None = None, aws_region_name: str | None = None, + aws_session_tags: Sequence[AwsSessionTag] | None = None, ) -> dict: """Handle same-account role assumption for IRSA.""" import boto3 @@ -1074,16 +1119,9 @@ class BaseAWSLLM: # Assume the role verbose_logger.debug("Attempting to assume role: %s with session: %s", aws_role_name, aws_session_name) - assume_role_params: Final = { - "RoleArn": aws_role_name, - "RoleSessionName": aws_session_name, - } - - # Add ExternalId parameter if provided - if aws_external_id is not None: - assume_role_params["ExternalId"] = aws_external_id - - return sts_client.assume_role(**assume_role_params) + return sts_client.assume_role( + **_assume_role_params(aws_role_name, aws_session_name, aws_external_id, aws_session_tags) + ) def _extract_credentials_and_ttl(self, sts_response: dict) -> tuple[Credentials, int | None]: """Extract credentials and TTL from STS response. @@ -1118,6 +1156,7 @@ class BaseAWSLLM: aws_region_name: str | None, aws_sts_endpoint: str | None, aws_external_id: str | None, + aws_session_tags: tuple[AwsSessionTag, ...] | None, ssl_verify: bool | str | None, ) -> tuple[Credentials, int | None]: """ @@ -1144,6 +1183,7 @@ class BaseAWSLLM: aws_region_name=aws_region_name, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=aws_session_tags, ssl_verify=ssl_verify, ) @@ -1159,6 +1199,7 @@ class BaseAWSLLM: aws_sts_endpoint: str | None = None, aws_external_id: str | None = None, ssl_verify: bool | str | None = None, + aws_session_tags: Sequence[AwsSessionTag] | None = None, ) -> tuple[Credentials, int | None]: """ Authenticate with AWS Role @@ -1189,6 +1230,7 @@ class BaseAWSLLM: aws_sts_endpoint=aws_sts_endpoint, ssl_verify=ssl_verify, aws_region_name=aws_region_name, + aws_session_tags=aws_session_tags, ) else: sts_response = self._handle_irsa_same_account( @@ -1198,6 +1240,7 @@ class BaseAWSLLM: aws_sts_endpoint=aws_sts_endpoint, ssl_verify=ssl_verify, aws_region_name=aws_region_name, + aws_session_tags=aws_session_tags, ) return self._extract_credentials_and_ttl(sts_response) @@ -1234,14 +1277,9 @@ class BaseAWSLLM: **sts_client_kwargs, ) - assume_role_params: Final = { - "RoleArn": aws_role_name, - "RoleSessionName": aws_session_name, - } - - # Add ExternalId parameter if provided - if aws_external_id is not None: - assume_role_params["ExternalId"] = aws_external_id + assume_role_params: Final = _assume_role_params( + aws_role_name, aws_session_name, aws_external_id, aws_session_tags + ) try: sts_response = sts_client.assume_role(**assume_role_params) @@ -1460,6 +1498,7 @@ class BaseAWSLLM: "aws_bedrock_runtime_endpoint", None ) # https://bedrock-runtime.{region_name}.amazonaws.com aws_external_id: Final = optional_params.pop("aws_external_id", None) + aws_session_tags: Final = optional_params.pop("aws_session_tags", None) if bearer_token is not None: return BearerRequestTarget( @@ -1478,6 +1517,7 @@ class BaseAWSLLM: aws_web_identity_token=aws_web_identity_token, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=aws_session_tags, ) return Boto3CredentialsInfo( credentials=credentials, @@ -1623,6 +1663,7 @@ class BaseAWSLLM: aws_web_identity_token: Final = optional_params.get("aws_web_identity_token", None) aws_sts_endpoint: Final = optional_params.get("aws_sts_endpoint", None) aws_external_id: Final = optional_params.get("aws_external_id", None) + aws_session_tags: Final = optional_params.get("aws_session_tags", None) aws_region_name: Final = self._get_aws_region_name(optional_params=optional_params, model=model) credentials: Final[Credentials] = self.get_credentials( @@ -1636,6 +1677,7 @@ class BaseAWSLLM: aws_web_identity_token=aws_web_identity_token, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=aws_session_tags, ) sigv4: Final = SigV4Auth(credentials, service_name, aws_region_name) diff --git a/litellm/llms/bedrock/batches/handler.py b/litellm/llms/bedrock/batches/handler.py index 4b500897642..f73d5f5d3cb 100644 --- a/litellm/llms/bedrock/batches/handler.py +++ b/litellm/llms/bedrock/batches/handler.py @@ -283,7 +283,7 @@ class BedrockBatchesHandler: ``aws_session_token``, ``aws_profile_name``, ``aws_role_name``, ``aws_session_name``, ``aws_web_identity_token``, ``aws_sts_endpoint``, - ``aws_external_id``). Unknown keys are ignored. + ``aws_external_id``, ``aws_session_tags``). Unknown keys are ignored. Returns: ``LiteLLMBatch`` shaped like an OpenAI Batch resource. @@ -317,6 +317,7 @@ class BedrockBatchesHandler: aws_web_identity_token=kwargs.get("aws_web_identity_token"), aws_sts_endpoint=kwargs.get("aws_sts_endpoint"), aws_external_id=kwargs.get("aws_external_id"), + aws_session_tags=kwargs.get("aws_session_tags"), ) client: Final = boto3.client( diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 984ba371898..573fc629383 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -355,6 +355,7 @@ class BedrockConverseLLM(BaseAWSLLM): aws_web_identity_token: Final = optional_params.pop("aws_web_identity_token", None) aws_sts_endpoint: Final = optional_params.pop("aws_sts_endpoint", None) aws_external_id: Final = optional_params.pop("aws_external_id", None) + aws_session_tags: Final = optional_params.pop("aws_session_tags", None) optional_params.pop("aws_region_name", None) litellm_params["aws_region_name"] = aws_region_name # [DO NOT DELETE] important for async calls @@ -373,6 +374,7 @@ class BedrockConverseLLM(BaseAWSLLM): aws_web_identity_token=aws_web_identity_token, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=aws_session_tags, ) ) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index be4f0f32689..cb2c70e74c8 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -93,6 +93,7 @@ _BEDROCK_AWS_AUTH_PARAMETER_KEYS: Final[tuple[str, ...]] = ( "aws_web_identity_token", "aws_sts_endpoint", "aws_external_id", + "aws_session_tags", ) @@ -1663,6 +1664,7 @@ class CommonBatchFilesUtils: aws_web_identity_token=optional_params.get("aws_web_identity_token"), aws_sts_endpoint=optional_params.get("aws_sts_endpoint"), aws_external_id=optional_params.get("aws_external_id"), + aws_session_tags=optional_params.get("aws_session_tags"), ) # Prepare the request data diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index e249feb9ff3..8ddc67c7f0e 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -73,6 +73,7 @@ class BedrockEmbedding(BaseAWSLLM): aws_web_identity_token: Final = optional_params.pop("aws_web_identity_token", None) aws_sts_endpoint: Final = optional_params.pop("aws_sts_endpoint", None) aws_external_id: Final = optional_params.pop("aws_external_id", None) + aws_session_tags: Final = optional_params.pop("aws_session_tags", None) ### SET REGION NAME ### if aws_region_name is None: @@ -103,6 +104,7 @@ class BedrockEmbedding(BaseAWSLLM): aws_web_identity_token=aws_web_identity_token, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=aws_session_tags, ) ) return credentials, aws_region_name diff --git a/litellm/llms/sagemaker/chat/handler.py b/litellm/llms/sagemaker/chat/handler.py index 3f62b7276df..38978300c52 100644 --- a/litellm/llms/sagemaker/chat/handler.py +++ b/litellm/llms/sagemaker/chat/handler.py @@ -36,6 +36,7 @@ class SagemakerChatHandler(BaseAWSLLM): aws_web_identity_token: Final = optional_params.pop("aws_web_identity_token", None) aws_sts_endpoint: Final = optional_params.pop("aws_sts_endpoint", None) aws_external_id: Final = optional_params.pop("aws_external_id", None) + aws_session_tags: Final = optional_params.pop("aws_session_tags", None) ### SET REGION NAME ### if aws_region_name is None: @@ -63,6 +64,7 @@ class SagemakerChatHandler(BaseAWSLLM): aws_web_identity_token=aws_web_identity_token, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=aws_session_tags, ) return credentials, aws_region_name diff --git a/litellm/llms/sagemaker/completion/handler.py b/litellm/llms/sagemaker/completion/handler.py index fb8074d3682..fad0a460647 100644 --- a/litellm/llms/sagemaker/completion/handler.py +++ b/litellm/llms/sagemaker/completion/handler.py @@ -59,6 +59,7 @@ class SagemakerLLM(BaseAWSLLM): aws_web_identity_token: Final = optional_params.pop("aws_web_identity_token", None) aws_sts_endpoint: Final = optional_params.pop("aws_sts_endpoint", None) aws_external_id: Final = optional_params.pop("aws_external_id", None) + aws_session_tags: Final = optional_params.pop("aws_session_tags", None) ### SET REGION NAME ### if aws_region_name is None: @@ -86,6 +87,7 @@ class SagemakerLLM(BaseAWSLLM): aws_web_identity_token=aws_web_identity_token, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=aws_session_tags, ) return credentials, aws_region_name diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index f78c4221f5a..f498f2fe8f5 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -316,6 +316,7 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( "aws_profile_name", "aws_session_name", "aws_external_id", + "aws_session_tags", "vertex_credentials", # Azure managed-identity / federated-auth token. The Azure provider # transformer reads ``azure_ad_token`` (top-level or via diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index 9f93886a9c6..76756ac35bb 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -1107,6 +1107,11 @@ class BedrockTag(TypedDict): value: str +class AwsSessionTag(TypedDict): + Key: str # writable-ok: boto3's STS stubs type assume_role Tags as writable TagTypeDef, which rejects ReadOnly + Value: str # writable-ok: boto3's STS stubs type assume_role Tags as writable TagTypeDef, which rejects ReadOnly + + class BedrockCreateBatchRequest(TypedDict, total=False): """ Request structure for creating a Bedrock batch inference job. diff --git a/litellm/types/router.py b/litellm/types/router.py index 5c9eab30f3d..30ee68f5545 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,7 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints @@ -21,6 +21,7 @@ if TYPE_CHECKING: from .completion import CompletionRequest from .embedding import EmbeddingRequest +from .llms.bedrock import AwsSessionTag from .llms.openai import OpenAIFileObject from .search import SearchProvider from .utils import ( @@ -288,6 +289,7 @@ class CredentialLiteLLMParams(BaseModel): aws_web_identity_token: str | None = None aws_sts_endpoint: str | None = None aws_external_id: str | None = None + aws_session_tags: Sequence[AwsSessionTag] | None = None aws_bedrock_runtime_endpoint: str | None = None aws_bedrock_project_id: str | None = None s3_bucket_name: str | None = None diff --git a/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py b/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py index dad2fdbf065..6c059423f74 100644 --- a/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py +++ b/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py @@ -194,6 +194,7 @@ class DummyCredentials: ("aws_web_identity_token", "dummy_web_identity_token"), ("aws_sts_endpoint", "dummy_sts_endpoint"), ("aws_external_id", "dummy_external_id"), + ("aws_session_tags", [{"Key": "team", "Value": "genai"}]), ], ) def test_dynamic_aws_params_propagation(model, param_name, param_value): diff --git a/tests/test_litellm/llms/bedrock/batches/test_handler.py b/tests/test_litellm/llms/bedrock/batches/test_handler.py index 2a9b7a6d138..4ccddc33e77 100644 --- a/tests/test_litellm/llms/bedrock/batches/test_handler.py +++ b/tests/test_litellm/llms/bedrock/batches/test_handler.py @@ -8,7 +8,7 @@ the tests don't hit AWS. from __future__ import annotations -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, patch import pytest @@ -480,3 +480,53 @@ def test_litellm_cancel_batch_dispatches_to_bedrock(patched_boto3): fake_client.stop_model_invocation_job.assert_called_once_with(jobIdentifier=JOB_ARN) assert batch.status == "cancelled" + + +def test_handle_model_invocation_job_status_builds_the_client_from_the_tagged_session(monkeypatch): + """Status polling must assume the role with the deployment's session tags, like every other call.""" + from botocore.exceptions import ClientError + + monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False) + monkeypatch.delenv("AWS_ROLE_ARN", raising=False) + tags = [{"Key": "team", "Value": "genai"}] + bedrock_client_kwargs: list[dict] = [] + fake_bedrock = MagicMock() + fake_bedrock.get_model_invocation_job.return_value = _fake_boto3_response() + + class FakeSTSClient: + def get_caller_identity(self): + return {"Arn": "arn:aws:iam::111111111111:user/litellm-proxy-pod"} + + def assume_role(self, **params): + if list(params.get("Tags", ())) != tags: + raise ClientError( + {"Error": {"Code": "AccessDenied", "Message": "is not authorized to perform: sts:TagSession"}}, + "AssumeRole", + ) + return { + "Credentials": { + "AccessKeyId": "ASIABATCHSTATUSTAGGED", + "SecretAccessKey": "assumed-secret", + "SessionToken": "assumed-session-token", + "Expiration": datetime.now(timezone.utc) + timedelta(minutes=30), + } + } + + def boto3_client(service_name, **kwargs): + if service_name == "sts": + return FakeSTSClient() + bedrock_client_kwargs.append(kwargs) + return fake_bedrock + + with patch("boto3.client", side_effect=boto3_client): + batch = BedrockBatchesHandler._handle_model_invocation_job_status( + batch_id=JOB_ARN, + aws_access_key_id="AKIABATCHSTATUSCALLER", + aws_secret_access_key="pod-caller-secret", + aws_role_name="arn:aws:iam::999999999999:role/litellm-batch-role", + aws_session_name="litellm-batch-session", + aws_session_tags=tags, + ) + + assert batch.status == "completed" + assert [kwargs["aws_access_key_id"] for kwargs in bedrock_client_kwargs] == ["ASIABATCHSTATUSTAGGED"] diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_moonshot_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_moonshot_transformation.py index 531f334e460..bcd1df26020 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_moonshot_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_moonshot_transformation.py @@ -15,6 +15,7 @@ AWS_AUTH_PARAMS = { "aws_sts_endpoint": "https://sts.us-west-2.amazonaws.com", "aws_bedrock_runtime_endpoint": "https://bedrock-runtime.us-west-2.amazonaws.com", "aws_external_id": "external", + "aws_session_tags": [{"Key": "team", "Value": "genai"}], } diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index cbf160c451f..bcba4bf7711 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -57,6 +57,7 @@ def test_aws_params_filtered_from_request_body(): "aws_sts_endpoint": "https://sts.amazonaws.com", "aws_bedrock_runtime_endpoint": "https://bedrock-runtime.us-west-2.amazonaws.com", "aws_external_id": "external-id-123", + "aws_session_tags": [{"Key": "team", "Value": "genai"}], } # Transform the request @@ -105,6 +106,9 @@ def test_aws_params_filtered_from_request_body(): assert ( "aws_external_id" not in result_json ), "AWS external ID should not be in request body" + assert ( + "aws_session_tags" not in result_json + ), "AWS session tags should not be in request body" # Also check that the sensitive values themselves are not in the response assert ( diff --git a/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py index f34b8eb1fb9..92bd6d48aae 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -6,12 +6,15 @@ extension, and AWS credential resolution is stubbed so nothing reaches STS. from __future__ import annotations +from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, patch +import boto3 import httpx import pytest from botocore.credentials import Credentials +from botocore.exceptions import ClientError from litellm.llms.bedrock.chat.converse_handler import BedrockConverseLLM from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.rust_bridge import chat_completions as bridge @@ -541,3 +544,54 @@ def test_bearer_token_auth_never_runs_the_sigv4_credential_chain(monkeypatch, co assert response.choices[0].message.content == "hi" assert client.post.call_args.kwargs["headers"]["Authorization"] == "Bearer bedrock-bearer-token" + + +def test_session_tags_sign_the_request_and_stay_out_of_the_body(monkeypatch): + """The tagged STS session signs the Converse call and the tags never reach the request body (#34069).""" + monkeypatch.setenv("LITELLM_RUST", "0") + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False) + monkeypatch.delenv("AWS_ROLE_ARN", raising=False) + tags = [{"Key": "team", "Value": "genai"}] + + class FakeSTSClient: + def get_caller_identity(self): + return {"Arn": "arn:aws:iam::111111111111:user/litellm-proxy-pod"} + + def assume_role(self, **params): + if list(params.get("Tags", ())) != tags: + raise ClientError( + {"Error": {"Code": "AccessDenied", "Message": "is not authorized to perform: sts:TagSession"}}, + "AssumeRole", + ) + return { + "Credentials": { + "AccessKeyId": "ASIACONVERSETAGGED", + "SecretAccessKey": "assumed-secret", + "SessionToken": "assumed-session-token", + "Expiration": datetime.now(timezone.utc) + timedelta(minutes=30), + } + } + + client = _sync_client_returning_converse_response() + with patch.object(boto3, "client", return_value=FakeSTSClient()): + response = BedrockConverseLLM().completion( + **_completion_kwargs( + optional_params={ + "maxTokens": 16, + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIACONVERSECALLER", + "aws_secret_access_key": "pod-caller-secret", + "aws_role_name": "arn:aws:iam::999999999999:role/litellm-converse-role", + "aws_session_name": "litellm-converse-session", + "aws_session_tags": tags, + }, + litellm_params={}, + client=client, + ) + ) + + assert response.choices[0].message.content == "hi" + sent = client.post.call_args.kwargs + assert "Credential=ASIACONVERSETAGGED/" in sent["headers"]["Authorization"] + assert "aws_session_tags" not in sent["data"] diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py index bcd1a29d0e8..1f8c9e77165 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py @@ -1036,6 +1036,63 @@ def test_load_credentials_assumes_role_with_external_id(monkeypatch): assert "aws_external_id" not in optional_params +def test_embedding_session_tags_sign_the_request_and_stay_out_of_the_body(monkeypatch): + """The tagged STS session signs the InvokeModel call and the tags never reach the body (#34069).""" + import datetime + + import boto3 + from botocore.exceptions import ClientError + + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False) + monkeypatch.delenv("AWS_ROLE_ARN", raising=False) + tags = [{"Key": "team", "Value": "genai"}] + + class FakeSTSClient: + def get_caller_identity(self): + return {"Arn": "arn:aws:iam::111111111111:user/litellm-proxy-pod"} + + def assume_role(self, **params): + if list(params.get("Tags", ())) != tags: + raise ClientError( + {"Error": {"Code": "AccessDenied", "Message": "is not authorized to perform: sts:TagSession"}}, + "AssumeRole", + ) + return { + "Credentials": { + "AccessKeyId": "ASIAEMBEDTAGGED", + "SecretAccessKey": "assumed-secret", + "SessionToken": "assumed-session-token", + "Expiration": datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=30), + } + } + + client = HTTPHandler() + with patch.object(boto3, "client", return_value=FakeSTSClient()), patch.object(client, "post") as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(titan_embedding_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + response = litellm.embedding( + model="bedrock/amazon.titan-embed-text-v1", + input=test_input, + client=client, + aws_region_name="us-east-1", + aws_access_key_id="AKIAEMBEDCALLERKEY", + aws_secret_access_key="pod-caller-secret", + aws_role_name="arn:aws:iam::999999999999:role/litellm-embed-role", + aws_session_name="litellm-embed-session", + aws_session_tags=tags, + ) + + assert response.data[0]["embedding"] == titan_embedding_response["embedding"] + sent = mock_post.call_args.kwargs + assert "Credential=ASIAEMBEDTAGGED/" in sent["headers"]["Authorization"] + assert "aws_session_tags" not in sent["data"] + + def test_bedrock_embedding_bearer_token_never_runs_the_sigv4_credential_chain(monkeypatch): """The deployment's AWS profile does not exist, so resolving SigV4 credentials raises; a bearer-token deployment must still serve the request, since the diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index f854d806bdc..1aa9cb3a9db 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -15,7 +15,7 @@ from unittest.mock import MagicMock, patch from botocore.awsrequest import AWSPreparedRequest, AWSRequest from botocore.auth import SigV4Auth from botocore.credentials import Credentials -from botocore.exceptions import NoCredentialsError +from botocore.exceptions import ClientError, NoCredentialsError import litellm from litellm.llms.bedrock.base_aws_llm import ( @@ -2395,6 +2395,283 @@ def test_assume_role_without_external_id(): ) +_SESSION_TAGS = ({"Key": "team", "Value": "genai"}, {"Key": "env", "Value": "prod"}) +_SORTED_SESSION_TAGS = ({"Key": "env", "Value": "prod"}, {"Key": "team", "Value": "genai"}) +_TAGGED_ROLE_ARN = "arn:aws:iam::123456789012:role/TaggedRole" + + +class _TagAwareSTSClient: + """STS stand-in for a trust policy that only admits sessions carrying exactly the expected tags.""" + + def __init__(self, expected_tags: tuple = (), access_key: str = "ASIATAGGEDSESSION") -> None: + self.expected_tags = expected_tags + self.access_key = access_key + self.assume_role_calls: list = [] + + def get_caller_identity(self): + return {"Arn": "arn:aws:iam::111111111111:user/litellm-proxy-pod"} + + def assume_role_with_web_identity(self, **params): + return { + "Credentials": { + "AccessKeyId": "ASIAIRSATEMP", + "SecretAccessKey": "irsa-temp-secret-key", + "SessionToken": "irsa-temp-session-token", + "Expiration": datetime.now(timezone.utc) + timedelta(hours=1), + } + } + + def assume_role(self, **params): + self.assume_role_calls.append(params) + if tuple(params.get("Tags", ())) != self.expected_tags: + raise ClientError( + {"Error": {"Code": "AccessDenied", "Message": "is not authorized to perform: sts:TagSession"}}, + "AssumeRole", + ) + return { + "Credentials": { + "AccessKeyId": self.access_key, + "SecretAccessKey": "assumed-secret-key", + "SessionToken": "assumed-session-token", + "Expiration": datetime.now(timezone.utc) + timedelta(hours=1), + } + } + + +def _irsa_env(tmp_path, irsa_role_arn: str) -> dict: + token_file = tmp_path / "web-identity-token" + token_file.write_text("test-web-identity-token") + return { + "AWS_WEB_IDENTITY_TOKEN_FILE": str(token_file), + "AWS_ROLE_ARN": irsa_role_arn, + "AWS_REGION": "us-east-1", + } + + +def test_assume_role_sends_session_tags(): + """The STS session carries the configured tags, so a trust policy gated on sts:TagSession admits it.""" + sts = _TagAwareSTSClient(expected_tags=_SESSION_TAGS) + + with patch("boto3.client", return_value=sts): + credentials, _ttl = BaseAWSLLM()._auth_with_aws_role( + aws_access_key_id=None, + aws_secret_access_key=None, + aws_session_token=None, + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="test-session", + aws_session_tags=list(_SESSION_TAGS), + ) + + assert credentials.access_key == "ASIATAGGEDSESSION" + assert sts.assume_role_calls == [ + {"RoleArn": _TAGGED_ROLE_ARN, "RoleSessionName": "test-session", "Tags": _SESSION_TAGS} + ] + + +def test_assume_role_sends_session_tags_alongside_external_id(): + sts = _TagAwareSTSClient(expected_tags=_SESSION_TAGS) + + with patch("boto3.client", return_value=sts): + credentials, _ttl = BaseAWSLLM()._auth_with_aws_role( + aws_access_key_id=None, + aws_secret_access_key=None, + aws_session_token=None, + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="test-session", + aws_external_id="UniqueExternalID123", + aws_session_tags=_SESSION_TAGS, + ) + + assert credentials.access_key == "ASIATAGGEDSESSION" + assert sts.assume_role_calls == [ + { + "RoleArn": _TAGGED_ROLE_ARN, + "RoleSessionName": "test-session", + "ExternalId": "UniqueExternalID123", + "Tags": _SESSION_TAGS, + } + ] + + +@pytest.mark.parametrize("aws_session_tags", [None, [], ()], ids=["none", "empty-list", "empty-tuple"]) +def test_assume_role_omits_the_tags_key_without_session_tags(aws_session_tags): + """Nothing configured means the AssumeRole request looks exactly as it did before tags existed.""" + sts = _TagAwareSTSClient(expected_tags=()) + + with patch("boto3.client", return_value=sts): + credentials, _ttl = BaseAWSLLM()._auth_with_aws_role( + aws_access_key_id=None, + aws_secret_access_key=None, + aws_session_token=None, + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="test-session", + aws_session_tags=aws_session_tags, + ) + + assert credentials.access_key == "ASIATAGGEDSESSION" + assert sts.assume_role_calls == [{"RoleArn": _TAGGED_ROLE_ARN, "RoleSessionName": "test-session"}] + + +def test_irsa_cross_account_assume_role_sends_session_tags(tmp_path): + irsa_role_arn = "arn:aws:iam::111111111111:role/eks-service-account-role" + sts = _TagAwareSTSClient(expected_tags=_SESSION_TAGS) + + with patch.dict(os.environ, _irsa_env(tmp_path, irsa_role_arn)), patch("boto3.client", return_value=sts): + credentials, _ttl = BaseAWSLLM()._auth_with_aws_role( + aws_access_key_id=None, + aws_secret_access_key=None, + aws_session_token=None, + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="test-session", + aws_session_tags=_SESSION_TAGS, + ) + + assert credentials.access_key == "ASIATAGGEDSESSION" + assert sts.assume_role_calls == [ + {"RoleArn": _TAGGED_ROLE_ARN, "RoleSessionName": "test-session", "Tags": _SESSION_TAGS} + ] + + +def test_irsa_same_account_assume_role_sends_session_tags(tmp_path): + sts = _TagAwareSTSClient(expected_tags=_SESSION_TAGS) + + with patch.dict(os.environ, _irsa_env(tmp_path, _TAGGED_ROLE_ARN)), patch("boto3.client", return_value=sts): + credentials, _ttl = BaseAWSLLM()._auth_with_aws_role( + aws_access_key_id=None, + aws_secret_access_key=None, + aws_session_token=None, + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="test-session", + aws_session_tags=_SESSION_TAGS, + ) + + assert credentials.access_key == "ASIATAGGEDSESSION" + assert sts.assume_role_calls == [ + {"RoleArn": _TAGGED_ROLE_ARN, "RoleSessionName": "test-session", "Tags": _SESSION_TAGS} + ] + + +def test_get_credentials_canonicalizes_session_tag_order_for_the_cache(): + """Two deployments listing the same tags in a different order share one STS session.""" + base_aws_llm = BaseAWSLLM() + sts = _TagAwareSTSClient(expected_tags=_SORTED_SESSION_TAGS) + + with patch.dict(os.environ, _os_environ_without_aws_keys(), clear=True), patch("boto3.client", return_value=sts): + first = base_aws_llm.get_credentials( + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="team-session", + aws_session_tags=list(_SESSION_TAGS), + ) + second = base_aws_llm.get_credentials( + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="team-session", + aws_session_tags=list(reversed(_SESSION_TAGS)), + ) + + assert first.access_key == second.access_key == "ASIATAGGEDSESSION" + assert sts.assume_role_calls == [ + {"RoleArn": _TAGGED_ROLE_ARN, "RoleSessionName": "team-session", "Tags": _SORTED_SESSION_TAGS} + ] + + +def test_get_credentials_scopes_the_cache_per_session_tag_set(): + """Different tag sets are different principals to AWS, so each gets its own STS session.""" + base_aws_llm = BaseAWSLLM() + mock_sts_client = _assume_role_sts_mock() + mock_sts_client.assume_role.side_effect = [ + { + "Credentials": { + "AccessKeyId": f"assumed-access-key-{team}", + "SecretAccessKey": "assumed-secret-key", + "SessionToken": f"assumed-session-token-{team}", + "Expiration": datetime.now(timezone.utc) + timedelta(hours=1), + } + } + for team in ("genai", "platform") + ] + + with ( + patch.dict(os.environ, _os_environ_without_aws_keys(), clear=True), + patch("boto3.client", return_value=mock_sts_client), + ): + genai = base_aws_llm.get_credentials( + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="team-session", + aws_session_tags=[{"Key": "team", "Value": "genai"}], + ) + platform = base_aws_llm.get_credentials( + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="team-session", + aws_session_tags=[{"Key": "team", "Value": "platform"}], + ) + + assert genai.access_key == "assumed-access-key-genai" + assert platform.access_key == "assumed-access-key-platform" + assert [call.kwargs["Tags"] for call in mock_sts_client.assume_role.call_args_list] == [ + ({"Key": "team", "Value": "genai"},), + ({"Key": "team", "Value": "platform"},), + ] + + +@pytest.mark.parametrize( + "aws_session_tags", + [ + "team=genai", + {"team": "genai"}, + [["team", "genai"]], + [{"key": "team", "value": "genai"}], + [{"Key": "team"}], + [{"Key": 1, "Value": "genai"}], + ], + ids=["string", "flat-dict", "pair-list", "lowercase-keys", "missing-value", "non-string-key"], +) +def test_get_credentials_rejects_malformed_session_tags(aws_session_tags): + with pytest.raises(ValueError, match="Invalid 'aws_session_tags' value"): + BaseAWSLLM().get_credentials( + aws_role_name=_TAGGED_ROLE_ARN, + aws_session_name="team-session", + aws_session_tags=aws_session_tags, + ) + + +def test_get_boto_credentials_from_optional_params_consumes_session_tags(): + """Tags feed the STS call and must not linger in optional_params to be serialized into the body.""" + sts = _TagAwareSTSClient(expected_tags=_SORTED_SESSION_TAGS) + optional_params = { + "aws_region_name": "us-east-1", + "aws_role_name": _TAGGED_ROLE_ARN, + "aws_session_name": "team-session", + "aws_session_tags": list(_SESSION_TAGS), + } + + with patch.dict(os.environ, _os_environ_without_aws_keys(), clear=True), patch("boto3.client", return_value=sts): + target = BaseAWSLLM()._get_boto_credentials_from_optional_params(optional_params) + + assert target.credentials.access_key == "ASIATAGGEDSESSION" + assert "aws_session_tags" not in optional_params + + +def test_sign_request_signs_with_the_tagged_sts_session(): + sts = _TagAwareSTSClient(expected_tags=_SORTED_SESSION_TAGS) + optional_params = { + "aws_region_name": "us-east-1", + "aws_role_name": _TAGGED_ROLE_ARN, + "aws_session_name": "team-session", + "aws_session_tags": list(_SESSION_TAGS), + } + + with patch.dict(os.environ, _os_environ_without_aws_keys(), clear=True), patch("boto3.client", return_value=sts): + headers, _body = BaseAWSLLM()._sign_request( + service_name="bedrock", + headers={}, + optional_params=optional_params, + request_data={"prompt": "hi"}, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-opus-5/invoke", + ) + + assert "Credential=ASIATAGGEDSESSION/" in headers["Authorization"] + + def test_converse_handler_external_id_extraction(): """Test that BedrockConverseLLM properly extracts and passes aws_external_id parameter""" from litellm.llms.bedrock.chat.converse_handler import BedrockConverseLLM diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index 3f03305423a..a8a21e2cd37 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -486,6 +486,7 @@ def test_merge_bedrock_aws_request_params_strips_caller_identity_when_deployment "aws_role_name": "arn:aws:iam::123456789012:role/caller", "aws_session_token": "caller-token", "aws_web_identity_token": "caller-web-identity", + "aws_session_tags": [{"Key": "team", "Value": "caller-chosen"}], "timeout": 600, }, ) @@ -500,6 +501,7 @@ def test_merge_bedrock_aws_request_params_strips_caller_identity_when_deployment "aws_role_name", "aws_session_token", "aws_web_identity_token", + "aws_session_tags", ): assert stripped not in merged @@ -616,6 +618,60 @@ def test_sign_aws_request_assumes_role_with_external_id(monkeypatch): assert signed_data == b'{"jobName": "litellm-batch-job"}' +def test_sign_aws_request_assumes_role_with_session_tags(monkeypatch): + """Batch and file signing must carry the deployment's session tags into the AssumeRole call too.""" + import datetime + from unittest.mock import patch + + import boto3 + from botocore.exceptions import ClientError + + from litellm.llms.bedrock.common_utils import CommonBatchFilesUtils + + monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False) + monkeypatch.delenv("AWS_ROLE_ARN", raising=False) + tags = [{"Key": "team", "Value": "genai"}] + + class FakeSTSClient: + def get_caller_identity(self): + return {"Arn": "arn:aws:iam::111111111111:user/litellm-proxy-pod"} + + def assume_role(self, **params): + if list(params.get("Tags", ())) != tags: + raise ClientError( + {"Error": {"Code": "AccessDenied", "Message": "is not authorized to perform: sts:TagSession"}}, + "AssumeRole", + ) + return { + "Credentials": { + "AccessKeyId": "ASIABATCHSIGNTAGGED", + "SecretAccessKey": "assumed-secret", + "SessionToken": "assumed-session-token", + "Expiration": datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=30), + } + } + + optional_params = { + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIABATCHSIGNCALLER", + "aws_secret_access_key": "pod-caller-secret", + "aws_role_name": "arn:aws:iam::999999999999:role/litellm-batch-sign-role", + "aws_session_name": "litellm-batch-sign-session", + "aws_session_tags": tags, + } + + with patch.object(boto3, "client", return_value=FakeSTSClient()): + signed_headers, _signed_data = CommonBatchFilesUtils().sign_aws_request( + service_name="bedrock", + data={"jobName": "litellm-batch-job"}, + endpoint_url="https://bedrock.us-east-1.amazonaws.com/model-invocation-job", + optional_params=optional_params, + ) + + authorization = {key.lower(): value for key, value in signed_headers.items()}["authorization"] + assert "Credential=ASIABATCHSIGNTAGGED/" in authorization + + # --------------------------------------------------------------------------- # # Provider error headers (LIT-5428) # # --------------------------------------------------------------------------- # diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_handler.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_handler.py index aa1a59d0e5c..bb891c06fa2 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_handler.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_handler.py @@ -46,3 +46,45 @@ def test_load_credentials_assumes_role_with_external_id(monkeypatch): assert credentials.token == "assumed-session-token" assert aws_region_name == "us-east-1" assert "aws_external_id" not in optional_params + + +def test_load_credentials_assumes_role_with_session_tags(monkeypatch): + """A trust policy gated on sts:TagSession only admits the session when the deployment's tags are sent.""" + monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False) + monkeypatch.delenv("AWS_ROLE_ARN", raising=False) + tags = [{"Key": "team", "Value": "genai"}] + + class FakeSTSClient: + def get_caller_identity(self): + return {"Arn": "arn:aws:iam::111111111111:user/litellm-proxy-pod"} + + def assume_role(self, **params): + if list(params.get("Tags", ())) != tags: + raise ClientError( + {"Error": {"Code": "AccessDenied", "Message": "is not authorized to perform: sts:TagSession"}}, + "AssumeRole", + ) + return { + "Credentials": { + "AccessKeyId": "ASIASMCHATTAGGED", + "SecretAccessKey": "assumed-secret", + "SessionToken": "assumed-session-token", + "Expiration": datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=30), + } + } + + optional_params = { + "aws_access_key_id": "AKIASMCHATCALLERKEY", + "aws_secret_access_key": "pod-caller-secret", + "aws_region_name": "us-east-1", + "aws_role_name": "arn:aws:iam::999999999999:role/litellm-sm-chat-role", + "aws_session_name": "litellm-sm-chat-session", + "aws_session_tags": tags, + } + + with patch.object(boto3, "client", return_value=FakeSTSClient()): + credentials, aws_region_name = SagemakerChatHandler()._load_credentials(optional_params) + + assert credentials.access_key == "ASIASMCHATTAGGED" + assert aws_region_name == "us-east-1" + assert "aws_session_tags" not in optional_params diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_completion_handler.py index 881bac096b1..9a5f36e2081 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_completion_handler.py @@ -219,3 +219,51 @@ def test_load_credentials_assumes_role_with_external_id(monkeypatch): assert credentials.token == "assumed-session-token" assert aws_region_name == "us-east-1" assert "aws_external_id" not in optional_params + + +def test_load_credentials_assumes_role_with_session_tags(monkeypatch): + """A trust policy gated on sts:TagSession only admits the session when the deployment's tags are sent.""" + import datetime + + import boto3 + from botocore.exceptions import ClientError + from unittest.mock import patch + + monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False) + monkeypatch.delenv("AWS_ROLE_ARN", raising=False) + tags = [{"Key": "team", "Value": "genai"}] + + class FakeSTSClient: + def get_caller_identity(self): + return {"Arn": "arn:aws:iam::111111111111:user/litellm-proxy-pod"} + + def assume_role(self, **params): + if list(params.get("Tags", ())) != tags: + raise ClientError( + {"Error": {"Code": "AccessDenied", "Message": "is not authorized to perform: sts:TagSession"}}, + "AssumeRole", + ) + return { + "Credentials": { + "AccessKeyId": "ASIASMCOMPTAGGED", + "SecretAccessKey": "assumed-secret", + "SessionToken": "assumed-session-token", + "Expiration": datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=30), + } + } + + optional_params = { + "aws_access_key_id": "AKIASMCOMPCALLERKEY", + "aws_secret_access_key": "pod-caller-secret", + "aws_region_name": "us-east-1", + "aws_role_name": "arn:aws:iam::999999999999:role/litellm-sm-completion-role", + "aws_session_name": "litellm-sm-completion-session", + "aws_session_tags": tags, + } + + with patch.object(boto3, "client", return_value=FakeSTSClient()): + credentials, aws_region_name = SagemakerLLM()._load_credentials(optional_params) + + assert credentials.access_key == "ASIASMCOMPTAGGED" + assert aws_region_name == "us-east-1" + assert "aws_session_tags" not in optional_params diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index aaf630ad29b..97cda730e04 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -3497,7 +3497,7 @@ class TestIsRequestBodySafeBlocksAwsIdentitySelectors: @pytest.mark.parametrize( "selector", - ["aws_profile_name", "aws_session_name", "aws_external_id"], + ["aws_profile_name", "aws_session_name", "aws_external_id", "aws_session_tags"], ) def test_aws_identity_selector_in_batch_body_is_rejected(self, selector): with pytest.raises(ValueError, match=selector): @@ -3516,7 +3516,7 @@ class TestIsRequestBodySafeBlocksAwsIdentitySelectors: @pytest.mark.parametrize( "selector", - ["aws_profile_name", "aws_session_name", "aws_external_id"], + ["aws_profile_name", "aws_session_name", "aws_external_id", "aws_session_tags"], ) def test_aws_identity_selector_under_extra_body_is_rejected(self, selector): with pytest.raises(ValueError, match=selector): diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py index fd933a9d993..7fcb76b638f 100644 --- a/tests/test_litellm/types/test_router.py +++ b/tests/test_litellm/types/test_router.py @@ -1,6 +1,7 @@ import logging import pytest +from pydantic import ValidationError from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, @@ -122,3 +123,26 @@ def test_drop_params_flags_and_strings_log_nothing(value, caplog): with caplog.at_level(logging.WARNING, logger="LiteLLM"): GenericLiteLLMParams(drop_params=value) assert caplog.text == "" + + +def test_aws_session_tags_round_trip_as_sts_shaped_pairs(): + """The deployment field keeps the exact Key/Value shape STS AssumeRole expects.""" + params = LiteLLM_Params( + model="bedrock/anthropic.claude-opus-5", + aws_session_tags=[{"Key": "team", "Value": "genai"}, {"Key": "env", "Value": "prod"}], + ) + + assert params.model_dump(exclude_none=True)["aws_session_tags"] == [ + {"Key": "team", "Value": "genai"}, + {"Key": "env", "Value": "prod"}, + ] + + +@pytest.mark.parametrize( + "aws_session_tags", + ["team=genai", {"team": "genai"}, [{"key": "team", "value": "genai"}], [{"Key": "team"}]], + ids=["string", "flat-dict", "lowercase-keys", "missing-value"], +) +def test_aws_session_tags_reject_shapes_sts_would_refuse(aws_session_tags): + with pytest.raises(ValidationError, match="aws_session_tags"): + LiteLLM_Params(model="bedrock/anthropic.claude-opus-5", aws_session_tags=aws_session_tags) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 83b0d58f2b2..1009655bedc 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23708,6 +23708,15 @@ export interface components { /** @description The decision record this request would have written to its log row */ routing_decision: components["schemas"]["StandardLoggingRoutingDecision"]; }; + /** AwsSessionTag */ + AwsSessionTag: { + /** Key */ + Key: string; + /** Value */ + Value: string; + } & { + [key: string]: unknown; + }; /** BaseLitellmParams */ BaseLitellmParams: { /** @@ -29426,6 +29435,8 @@ export interface components { aws_secret_access_key?: string | null; /** Aws Session Name */ aws_session_name?: string | null; + /** Aws Session Tags */ + aws_session_tags?: components["schemas"]["AwsSessionTag"][] | null; /** Aws Session Token */ aws_session_token?: string | null; /** Aws Sts Endpoint */ @@ -39617,6 +39628,8 @@ export interface components { aws_secret_access_key?: string | null; /** Aws Session Name */ aws_session_name?: string | null; + /** Aws Session Tags */ + aws_session_tags?: components["schemas"]["AwsSessionTag"][] | null; /** Aws Session Token */ aws_session_token?: string | null; /** Aws Sts Endpoint */ From 261777d633c6c5466a0711a7a889ca2cfb0a08d6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:40:29 -0700 Subject: [PATCH 207/411] fix(databricks): keep top-level reasoning_content from OpenAI-compatible gateway models The Databricks chat transformation only parsed reasoning out of FMAPI-style reasoning content blocks, so external models behind Databricks AI Gateway that return the OpenAI-style top-level reasoning_content string lost it, both in the final message and in every streamed delta. Fall back to the shared OpenAI reasoning helper when no reasoning block exists, and keep the delta's own reasoning_content when streaming. --- .../llms/databricks/chat/transformation.py | 33 ++++++-- litellm/types/llms/databricks.py | 9 ++ .../test_databricks_chat_transformation.py | 83 +++++++++++++++++++ 3 files changed, 118 insertions(+), 7 deletions(-) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index e2e2ea5b553..54c7040daf9 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo _should_convert_tool_call_to_json_mode, ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_reasoning_content, # pyright: ignore[reportPrivateUsage] # same import as the OpenAI transformation strip_litellm_internal_message_fields, strip_name_from_message, ) @@ -23,7 +24,9 @@ from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.databricks import ( AllDatabricksContentValues, DatabricksChoice, + DatabricksDelta, DatabricksFunction, + DatabricksMessage, DatabricksResponse, DatabricksTool, ) @@ -534,6 +537,19 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): thinking_blocks.append(thinking_block) return reasoning_content, thinking_blocks + @staticmethod + def extract_top_level_reasoning_content(delta: DatabricksDelta) -> str | None: + return delta.get("reasoning_content") + + @staticmethod + def resolve_reasoning_and_content( + message: DatabricksMessage, block_reasoning_content: str | None + ) -> tuple[str | None, str | None]: + content_str: Final = DatabricksConfig.extract_content_str(message["content"]) + if block_reasoning_content is not None: + return block_reasoning_content, content_str + return _extract_reasoning_content({**message, "content": content_str}) + @staticmethod def extract_citations( content: AllDatabricksContentValues | None, @@ -577,14 +593,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): finish_reason = "stop" if translated_message is None: - ## get the content str - content_str = DatabricksConfig.extract_content_str(choice["message"]["content"]) - - ## get the reasoning content ( - reasoning_content, + block_reasoning_content, thinking_blocks, ) = DatabricksConfig.extract_reasoning_content(choice["message"].get("content")) + reasoning_content, content_str = DatabricksConfig.resolve_reasoning_and_content( + choice["message"], block_reasoning_content + ) citations = DatabricksConfig.extract_citations(choice["message"].get("content")) @@ -738,12 +753,16 @@ class DatabricksChatResponseIterator(BaseModelResponseIterator): # extract the reasoning content ( - reasoning_content, + block_reasoning_content, thinking_blocks, ) = DatabricksConfig.extract_reasoning_content(choice["delta"].get("content")) choice["delta"]["content"] = content_str - choice["delta"]["reasoning_content"] = reasoning_content + choice["delta"]["reasoning_content"] = ( + block_reasoning_content + if block_reasoning_content is not None + else DatabricksConfig.extract_top_level_reasoning_content(choice["delta"]) + ) choice["delta"]["thinking_blocks"] = thinking_blocks translated_choices.append(choice) return ModelResponseStream( diff --git a/litellm/types/llms/databricks.py b/litellm/types/llms/databricks.py index e87a684aab8..a9c027bd2de 100644 --- a/litellm/types/llms/databricks.py +++ b/litellm/types/llms/databricks.py @@ -2,6 +2,7 @@ from typing import Any, Literal from pydantic import BaseModel from typing_extensions import ( + ReadOnly, Required, TypedDict, ) @@ -57,6 +58,14 @@ class DatabricksMessage(TypedDict, total=False): role: Required[str] content: Required[AllDatabricksContentValues] tool_calls: list[DatabricksTool] | None + reasoning_content: ReadOnly[str | None] + reasoning: ReadOnly[str | None] + + +class DatabricksDelta(TypedDict, total=False): + role: ReadOnly[str] + content: ReadOnly[AllDatabricksContentValues | None] + reasoning_content: ReadOnly[str | None] class DatabricksChoice(TypedDict, total=False): diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 02655fb7f77..89d1aa52838 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -590,3 +590,86 @@ def test_chunk_parser_without_usage_still_parses_content(): assert result.id == "chatcmpl-test" assert result.model == "databricks-claude-sonnet-5" assert result.choices[0]["delta"]["content"] == "hi" + + +@pytest.mark.parametrize("reasoning_key", ["reasoning_content", "reasoning"]) +def test_transform_choices_surfaces_top_level_reasoning_content(reasoning_key: str) -> None: + config = DatabricksConfig() + databricks_choices = [ + { + "message": { + "role": "assistant", + "content": "391", + reasoning_key: "We need answer just number. 17*23=391.", + }, + "index": 0, + "finish_reason": "stop", + } + ] + + choices = config._transform_dbrx_choices(choices=databricks_choices) + + assert choices[0].message.content == "391" + assert choices[0].message.reasoning_content == "We need answer just number. 17*23=391." + assert getattr(choices[0].message, "thinking_blocks", None) is None + + +def test_transform_choices_parses_think_tags_in_string_content(): + config = DatabricksConfig() + databricks_choices = [ + { + "message": {"role": "assistant", "content": "17 times 23391"}, + "index": 0, + "finish_reason": "stop", + } + ] + + choices = config._transform_dbrx_choices(choices=databricks_choices) + + assert choices[0].message.content == "391" + assert choices[0].message.reasoning_content == "17 times 23" + + +def test_transform_choices_prefers_reasoning_blocks_over_top_level_field(): + config = DatabricksConfig() + databricks_choices = [ + { + "message": { + "role": "assistant", + "content": [ + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "from block"}]}, + {"type": "text", "text": "391"}, + ], + "reasoning_content": "from field", + }, + "index": 0, + "finish_reason": "stop", + } + ] + + choices = config._transform_dbrx_choices(choices=databricks_choices) + + assert choices[0].message.reasoning_content == "from block" + assert choices[0].message.content == "391" + + +def test_chunk_parser_surfaces_top_level_reasoning_delta(): + iterator = DatabricksChatResponseIterator(None, sync_stream=True) + chunk = { + "id": "1", + "object": "chat.completion.chunk", + "created": 0, + "model": "lit-qa-deepseek-v4-flash", + "choices": [ + { + "delta": {"role": "assistant", "content": "", "reasoning_content": "We need answer"}, + "index": 0, + "finish_reason": None, + } + ], + } + + parsed = iterator.chunk_parser(chunk) + + assert parsed.choices[0].delta.reasoning_content == "We need answer" + assert parsed.choices[0].delta.content == "" From 751431dc3f782b1aa93908533dbc9d9e0062b400 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:41:38 -0700 Subject: [PATCH 208/411] fix(anthropic): strip bridge reasoning in the native messages transform, not the empty-block pass --- litellm/llms/anthropic/common_utils.py | 36 ++++++++--- .../messages/transformation.py | 3 +- ..._anthropic_messages_encrypted_reasoning.py | 50 ++++++++++++++++ .../anthropic/test_anthropic_common_utils.py | 59 +++++++++++++++---- 4 files changed, 126 insertions(+), 22 deletions(-) create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index f46227239d8..cb24f4153f9 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1202,6 +1202,30 @@ def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[A return out +def _without_encrypted_reasoning_blocks(message: dict) -> dict | None: # mutable-ok: Anthropic message payload shape + content: Final = message.get("content") + if not isinstance(content, list): + return message + kept: Final = [b for b in content if not is_encrypted_reasoning_block(b)] # mutable-ok: API message payload + if len(kept) == len(content): + return message + if not kept: + return None + return {**message, "content": kept} # mutable-ok: API message payload + + +def strip_encrypted_reasoning_blocks_from_anthropic_messages( + messages: Sequence[dict], # mutable-ok: Anthropic message payload shape +) -> list[dict]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + """ + Drop thinking / redacted_thinking blocks that carry another provider's encrypted + reasoning (a turn the Responses API bridge served) before the request reaches + Anthropic, which cannot verify them. Anthropic's own signed blocks are kept. + """ + stripped: Final = (_without_encrypted_reasoning_blocks(m) for m in messages) + return [m for m in stripped if m is not None] # mutable-ok: API message payload + + def strip_thinking_blocks_from_anthropic_messages_request_dict( data: dict[str, Any], ) -> None: @@ -1236,10 +1260,8 @@ def strip_empty_content_blocks_from_anthropic_messages( on the unified ``/v1/messages`` path. ``/v1/chat/completions`` already handles this in ``anthropic_messages_pt``; this helper provides the equivalent guarantee for the native Anthropic Messages path. - A thinking or ``redacted_thinking`` block whose signature or data carries - another provider's encrypted reasoning (a turn served by the Responses API - bridge) is dropped too, since Anthropic cannot verify it; every other - ``redacted_thinking`` block is left alone. + ``redacted_thinking`` blocks are never touched: they carry opaque + ``data`` instead of thinking text. Messages whose content is a list and becomes empty after stripping are omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`. @@ -1252,11 +1274,7 @@ def strip_empty_content_blocks_from_anthropic_messages( out.append(m) continue content = m["content"] - filtered = [ # mutable-ok: rebuilt message content list - b - for b in content - if not _is_empty_text_block(b) and not is_empty_thinking_block(b) and not is_encrypted_reasoning_block(b) - ] + filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_thinking_block(b)] if len(filtered) == len(content): out.append(m) elif filtered: diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 8267da157ad..9f9346fad4d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -25,6 +25,7 @@ from ...common_utils import ( AnthropicModelInfo, optionally_handle_anthropic_oauth, strip_advisor_blocks_from_messages, + strip_encrypted_reasoning_blocks_from_anthropic_messages, ) DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" @@ -613,7 +614,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): messages = strip_advisor_blocks_from_messages(messages) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( - messages=messages, + messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py new file mode 100644 index 00000000000..c64e9d392e5 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py @@ -0,0 +1,50 @@ +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, +) +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) + + +def _transform(messages): + return AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-5", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 1024}, + litellm_params={}, + headers={}, + ) + + +def test_reasoning_replayed_from_the_responses_bridge_never_reaches_anthropic(): + """Claude Code resumed on a Claude model echoes the thinking blocks a gpt turn produced.""" + messages = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + {"type": "text", "text": "The answer."}, + ], + }, + {"role": "user", "content": "And the next one?"}, + ] + request = _transform(messages) + assert request["messages"][1]["content"] == [{"type": "text", "text": "The answer."}] + assert len(messages[1]["content"]) == 3 + + +def test_anthropic_signed_thinking_blocks_are_forwarded_untouched(): + messages = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."}, + ], + }, + ] + assert _transform(messages)["messages"] == messages diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 336cbf57e29..97d22274b1c 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -42,12 +42,15 @@ FAKE_AUTH_TOKEN = "sk-ant-aut01-fake-auth-token-for-testing-123456789" def test_is_claude_code_one_shot_subagent_request(messages, system, expected): from litellm.llms.anthropic.common_utils import is_claude_code_one_shot_subagent_request - assert is_claude_code_one_shot_subagent_request( - messages=messages, - system=system, - tools=None, - user_agent="claude-cli/2.1.263 (external, cli)", - ) is expected + assert ( + is_claude_code_one_shot_subagent_request( + messages=messages, + system=system, + tools=None, + user_agent="claude-cli/2.1.263 (external, cli)", + ) + is expected + ) class TestOptionallyHandleAnthropicOAuth: @@ -1541,8 +1544,8 @@ class TestAnthropicThinkingSignatureSelfHeal: out = strip_empty_content_blocks_from_anthropic_messages(msgs) assert [b["type"] for b in out[0]["content"]] == ["thinking"] - def test_strip_drops_encrypted_reasoning_blocks_from_the_responses_bridge(self): - """A session resumed on an Anthropic model replays reasoning only OpenAI can verify.""" + def test_strip_keeps_encrypted_reasoning_blocks_for_the_responses_bridge(self): + """The /v1/messages handler runs this before dispatch, so the bridge must still see the replay.""" from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) @@ -1556,14 +1559,46 @@ class TestAnthropicThinkingSignatureSelfHeal: "content": [ {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, - {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, {"type": "text", "text": "The answer."}, ], } ] - out = strip_empty_content_blocks_from_anthropic_messages(msgs) - assert [b["type"] for b in out[0]["content"]] == ["redacted_thinking", "text"] - assert len(msgs[0]["content"]) == 4 + assert strip_empty_content_blocks_from_anthropic_messages(msgs) == msgs + + def test_strip_encrypted_reasoning_drops_only_the_bridge_tagged_blocks(self): + """A session resumed on an Anthropic model replays reasoning only OpenAI can verify.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, + ) + from litellm.llms.anthropic.common_utils import ( + strip_encrypted_reasoning_blocks_from_anthropic_messages, + ) + + msgs = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_3")}, + {"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."}, + ], + }, + ] + out = strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs) + assert [m["role"] for m in out] == ["user", "assistant"] + assert [b["type"] for b in out[1]["content"]] == ["thinking", "redacted_thinking", "text"] + assert out[1]["content"][0]["signature"] == "EqQBCkYIAxgCIkA_anthropic_signed" + assert len(msgs[1]["content"]) == 2 + assert len(msgs[2]["content"]) == 4 def test_strip_empty_text_blocks_treats_null_text_as_empty(self): from litellm.llms.anthropic.common_utils import ( From 420282acb7704772ffb47c2ed4f84d455a2a385d Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 19:55:02 +0000 Subject: [PATCH 209/411] fix(registry): set max_output_tokens on vertex_ai/xai/grok-4.3 and grok-4.6 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 664c2737157..885021692c9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48204,6 +48204,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 200000, + "max_output_tokens": 200000, "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, @@ -48222,6 +48223,7 @@ "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 524288, + "max_output_tokens": 524288, "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 664c2737157..885021692c9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48204,6 +48204,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 200000, + "max_output_tokens": 200000, "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, @@ -48222,6 +48223,7 @@ "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 524288, + "max_output_tokens": 524288, "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, From 17ca562b6a5d208d1b41a58933491936d4cefee2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:01:11 -0700 Subject: [PATCH 210/411] test(llm_translation): expect an empty choices list to convert instead of raising --- .../test_convert_dict_to_chat_completion.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index b6e30ddc711..09d1adba17c 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -1623,15 +1623,11 @@ class TestMissingChoicesGuard: assert "no 'choices'" in exc_info.value.message - def test_convert_to_model_response_object_empty_choices_raises_api_error(self): - """Empty choices list raises APIError, same as missing/null choices. + def test_convert_to_model_response_object_empty_choices_returns_empty_list(self): + """An empty choices list is a real provider answer, so it converts to choices=[] instead of raising. - Provider-specific repair (e.g. github_copilot synthesizing choices for - Anthropic-native responses) happens before this guard, in the provider - config; the core utility keeps treating empty choices as an error. + See: https://github.com/BerriAI/litellm/issues/40276 """ - from litellm.exceptions import APIError - response_object = { "id": "msg_123", "model": "some-model", @@ -1639,13 +1635,14 @@ class TestMissingChoicesGuard: "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, } - with pytest.raises(APIError) as exc_info: - convert_to_model_response_object( - response_object=response_object, - model_response_object=ModelResponse(), - ) + result = convert_to_model_response_object( + response_object=response_object, + model_response_object=ModelResponse(), + ) - assert "no 'choices'" in exc_info.value.message + assert isinstance(result, ModelResponse) + assert result.choices == [] + assert result.usage.prompt_tokens == 10 def test_convert_to_model_response_object_null_choices_raises_api_error(self): """choices=None raises APIError.""" From f3a2844080f5b2dd003a084eece2861e78a1870d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:05:11 -0700 Subject: [PATCH 211/411] test(convert_dict_to_response): keep the regression test locals final and comment-free --- .../test_convert_dict_to_response.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index ea50bff462e..c3e99f01cec 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -108,7 +108,7 @@ def test_convert_empty_choices_response() -> None: convert_to_streaming_response, ) - resp = { + resp: Final = { "id": "x", "created": 1, "model": "gemini-3.5-flash", @@ -117,7 +117,7 @@ def test_convert_empty_choices_response() -> None: "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, "vertex_ai_safety_results": ["blocked"], } - result = convert_to_model_response_object( + result: Final = convert_to_model_response_object( response_object=resp, model_response_object=ModelResponse(), response_type="completion", @@ -125,8 +125,7 @@ def test_convert_empty_choices_response() -> None: assert result.choices == [] assert getattr(result, "vertex_ai_safety_results") == ["blocked"] - # Test sync streaming generator handles empty choices - sync_stream = list(convert_to_streaming_response(response_object=resp)) + sync_stream: Final = list(convert_to_streaming_response(response_object=resp)) assert len(sync_stream) == 1 assert sync_stream[0].choices == [] @@ -137,7 +136,7 @@ async def test_convert_empty_choices_response_async() -> None: convert_to_streaming_response_async, ) - resp = { + resp: Final = { "id": "x", "created": 1, "model": "gemini-3.5-flash", @@ -145,9 +144,7 @@ async def test_convert_empty_choices_response_async() -> None: "choices": [], "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, } - async_chunks = [] - async for chunk in convert_to_streaming_response_async(response_object=resp): - async_chunks.append(chunk) + async_chunks: Final = [chunk async for chunk in convert_to_streaming_response_async(response_object=resp)] assert len(async_chunks) == 1 assert async_chunks[0].choices == [] @@ -155,7 +152,7 @@ async def test_convert_empty_choices_response_async() -> None: def test_convert_missing_choices_raises_api_error() -> None: from litellm.exceptions import APIError - resp = { + resp: Final = { "id": "x", "created": 1, "model": "gemini-3.5-flash", From fe242f87028a9bd8ae2fcd4e579436f32614c116 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:16:57 -0700 Subject: [PATCH 212/411] test(databricks): pin the streamed reasoning delta shape and alias --- .../databricks/chat/test_databricks_chat_transformation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 89d1aa52838..38d5844b9b9 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -653,7 +653,8 @@ def test_transform_choices_prefers_reasoning_blocks_over_top_level_field(): assert choices[0].message.content == "391" -def test_chunk_parser_surfaces_top_level_reasoning_delta(): +@pytest.mark.parametrize("reasoning_key", ["reasoning_content", "reasoning"]) +def test_chunk_parser_surfaces_top_level_reasoning_delta(reasoning_key: str) -> None: iterator = DatabricksChatResponseIterator(None, sync_stream=True) chunk = { "id": "1", @@ -662,7 +663,7 @@ def test_chunk_parser_surfaces_top_level_reasoning_delta(): "model": "lit-qa-deepseek-v4-flash", "choices": [ { - "delta": {"role": "assistant", "content": "", "reasoning_content": "We need answer"}, + "delta": {"role": "assistant", "content": None, reasoning_key: "We need answer"}, "index": 0, "finish_reason": None, } @@ -672,4 +673,4 @@ def test_chunk_parser_surfaces_top_level_reasoning_delta(): parsed = iterator.chunk_parser(chunk) assert parsed.choices[0].delta.reasoning_content == "We need answer" - assert parsed.choices[0].delta.content == "" + assert parsed.choices[0].delta.content is None From c4bd3763e87ededa07ac552bb5fab67691bd3d49 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:17:19 -0700 Subject: [PATCH 213/411] test(proxy): type the background retrieval governance tests The two test methods, the policy_engine fixture, and the two inner stubs in TestBackgroundResponseRetrievalGovernance now carry full parameter and return annotations, closing the Greptile thread that 94f9230d13 left open. --- .../proxy/test_common_request_processing.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index c451a3b4cb0..f119926e4f4 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -3,7 +3,7 @@ import copy import datetime import json from types import MappingProxyType, SimpleNamespace -from typing import AsyncGenerator, Callable, Final, Optional +from typing import AsyncGenerator, Callable, Final, Iterator, Optional from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -8303,7 +8303,7 @@ class TestBackgroundResponseRetrievalGovernance: GOVERNED_MODEL_ID = "deployment-governed" @pytest.fixture - def policy_engine(self): + def policy_engine(self) -> Iterator[None]: from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -8353,10 +8353,14 @@ class TestBackgroundResponseRetrievalGovernance: mock_request = MagicMock(spec=Request) mock_request.headers = {} - async def passthrough_add_litellm_data_to_request(data, **kwargs): + async def passthrough_add_litellm_data_to_request( + data: dict[str, object], **kwargs: object + ) -> dict[str, object]: return data - async def decrypting_pre_call_hook(user_api_key_dict, data, call_type): + async def decrypting_pre_call_hook( + user_api_key_dict: ProxyUserAPIKeyAuth, data: dict[str, object], call_type: str + ) -> dict[str, object]: if data.get("response_id") == client_facing_response_id: data["response_id"] = encoded_response_id return data @@ -8383,8 +8387,8 @@ class TestBackgroundResponseRetrievalGovernance: @pytest.mark.asyncio async def test_retrieving_a_background_response_attaches_its_model_post_call_pipeline( - self, policy_engine, monkeypatch - ): + self, policy_engine: None, monkeypatch: pytest.MonkeyPatch + ) -> None: data = await self._pre_call("aget_responses", monkeypatch) assert data["response_id"].startswith("resp_bGl0ZWxsbTpjdXN0b21f") @@ -8397,8 +8401,8 @@ class TestBackgroundResponseRetrievalGovernance: @pytest.mark.asyncio async def test_submitting_a_response_does_not_attach_pipelines_from_its_response_id( - self, policy_engine, monkeypatch - ): + self, policy_engine: None, monkeypatch: pytest.MonkeyPatch + ) -> None: data = await self._pre_call("aresponses", monkeypatch) assert "_guardrail_pipelines" not in data["litellm_metadata"] From 1bcb587d336c8a141809ca0fe165d1b305f1122f Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:32:18 -0700 Subject: [PATCH 214/411] fix(mcp): challenge gateway-owned per-server authentication --- .../mcp_server/auth/user_api_key_auth_mcp.py | 15 +---- .../mcp_server/discoverable_endpoints.py | 11 ++-- .../types/mcp_server/mcp_server_manager.py | 18 +++++- .../auth/test_user_api_key_auth_mcp.py | 38 ++++++----- .../mcp_server/test_discoverable_endpoints.py | 63 +++++++++---------- 5 files changed, 75 insertions(+), 70 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 4bac65125b4..7e6de474b0b 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -179,16 +179,7 @@ def _gateway_dcr_challenge_target( mcp_servers: list[str] | None, client_ip: str | None, ) -> str | None: - """The single path-named server this request targets, iff it resolves to a - gateway-managed oauth2 server — the one per-server shape the gateway's own keyless - DCR flow serves end to end, so the 401 challenge may advertise the per-server - protected-resource metadata (whose ``authorization_servers`` names the gateway). - - Multi-server CSV paths, header/path mismatches, unknown names, and every - client-forwarded or delegated mode return ``None``: those cells keep their existing - challenge (or absence of one), and a challenge is never emitted for a name the - public discovery routes would 404, so this reveals exactly the server set the - per-server protected-resource metadata already reveals.""" + """Resolve a single path target whose sign-in metadata advertises the gateway.""" from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -217,7 +208,7 @@ def _is_gateway_dcr_challenge_scope( the caller is not a cold-start DCR client), on the scopes the gateway's keyless flow serves: the aggregate ``/mcp`` endpoint, an ``x-mcp-servers``-scoped request (the resource the client configured is still ``/mcp``), or a per-server path whose - single target is a gateway-managed oauth2 server. Every other named target keeps + single target advertises gateway-owned sign-in. Every other named target keeps its existing behavior, failing closed to the original admission error.""" if not _is_litellm_auth_admission_error(exc): return False @@ -236,7 +227,7 @@ def _gateway_dcr_challenge( ) -> HTTPException: """The RFC 9728 challenge pointing the client at the protected-resource metadata matching the scope it requested: the per-server document (same URL spelling the - request arrived on) when the single target is a gateway-managed oauth2 server, + request arrived on) when the single target advertises gateway-owned sign-in, else the gateway's aggregate document. Either way the client discovers the gateway as its authorization server and starts the same sign-in flow. diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index cab4b6c161a..f18acb7d88d 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -2310,8 +2310,7 @@ async def _build_oauth_protected_resource_response( it. Only the legacy ``is_oauth_passthrough`` opt-in rewrites ``resource`` to the gateway's own URL so clients present the bearer token back to the gateway. - An explicitly named gateway-managed oauth2 server (interactive with - gateway-vaulted per-user tokens, or M2M) advertises the gateway's own + An explicitly named server with gateway-owned sign-in advertises the gateway's own authorization server (``{base}/mcp``): a keyless DCR client that configured the per-server URL completes the same sign-in flow the aggregate ``/mcp`` endpoint supports and is admitted with a gateway session bearer. The per-server relay @@ -2401,11 +2400,6 @@ async def _build_oauth_protected_resource_response( if obo_response is not None: return obo_response - # An OBO server with no configured issuer falls through to the gateway default so discovery still - # returns metadata; every other non-oauth2 named server 404s to avoid enumeration. - if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange: - _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth-protected resource") - if explicitly_named and mcp_server is not None and mcp_server.advertises_gateway_authorization_server: return { "authorization_servers": [f"{request_base_url}/mcp"], @@ -2413,6 +2407,9 @@ async def _build_oauth_protected_resource_response( "scopes_supported": (mcp_server.scopes if mcp_server.scopes else []), } + if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange: + _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth-protected resource") + return { "authorization_servers": [ (f"{request_base_url}/{mcp_server_name}" if mcp_server_name else f"{request_base_url}") diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 84ffd50eea1..ab2e8a70754 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -250,7 +250,23 @@ class MCPServer(BaseModel): @property def advertises_gateway_authorization_server(self) -> bool: """Whether named discovery should advertise the aggregate gateway authorization server.""" - return self.is_gateway_managed_oauth2 and not self.uses_per_server_oauth_relay + if self.auth_type == MCPAuth.oauth2: + return self.is_gateway_managed_oauth2 and not self.uses_per_server_oauth_relay + if self.auth_type not in ( + None, + MCPAuth.none, + MCPAuth.api_key, + MCPAuth.bearer_token, + MCPAuth.basic, + MCPAuth.authorization, + MCPAuth.token, + MCPAuth.aws_sigv4, + ): + return False + return not any( + header.lower() in ("authorization", "x-api-key", "api-key", "apikey") + for header in (self.extra_headers or ()) + ) @property def is_true_passthrough(self) -> bool: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 20f4719e4bf..e6c8d4ee039 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -7193,14 +7193,13 @@ class TestAggregateGatewayDcrChallenge: www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] assert www_authenticate == f"Bearer {self._EXPECTED_RESOURCE_METADATA}" - async def test_per_server_challenge_for_gateway_managed_oauth2(self): - """Anonymous request to a per-server path whose single target is a gateway-managed - oauth2 server: 401 plus the RFC 9728 challenge advertising the PER-SERVER - protected-resource metadata in the same URL spelling the request used, so a keyless - DCR client configured with either per-server spelling discovers the gateway as the - authorization server (LIT-4864). Covers interactive and M2M, which the gateway can - both serve end to end.""" - from litellm.types.mcp import MCPAuth + @pytest.mark.parametrize( + "auth_type", + (None, "none", "api_key", "bearer_token", "basic", "aws_sigv4", "authorization", "token", "oauth2"), + ) + @pytest.mark.parametrize("bearer_presented", (False, True)) + async def test_per_server_challenge_for_gateway_owned_auth(self, auth_type, bearer_presented): + """Gateway admission challenges are independent of upstream authentication.""" from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -7209,7 +7208,7 @@ class TestAggregateGatewayDcrChallenge: server_name="github", url="https://upstream.example/mcp", transport="http", - auth_type=MCPAuth.oauth2, + auth_type=auth_type, ) for path, expected_metadata_path in ( ("/mcp/github", "/.well-known/oauth-protected-resource/mcp/github"), @@ -7223,10 +7222,16 @@ class TestAggregateGatewayDcrChallenge: ): mock_mgr.get_mcp_server_by_name.return_value = server with pytest.raises(HTTPException) as exc_info: - await MCPRequestHandler.process_mcp_request(self._scope(path=path)) + await MCPRequestHandler.process_mcp_request( + self._scope( + path=path, + extra_headers=((b"authorization", b"Bearer invalid-key"),) if bearer_presented else (), + ) + ) assert exc_info.value.status_code == 401 www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] - assert www_authenticate == f'Bearer resource_metadata="http://testserver{expected_metadata_path}"' + error = 'error="invalid_token", ' if bearer_presented else "" + assert www_authenticate == f'Bearer {error}resource_metadata="http://testserver{expected_metadata_path}"' async def test_per_server_challenge_keeps_spelling_under_server_root_path(self): """On a sub-path deployment the challenge must still advertise the spelling the client @@ -7303,10 +7308,7 @@ class TestAggregateGatewayDcrChallenge: ) def test_challenge_target_excludes_every_non_gateway_managed_mode(self): - """Unit pin of the challenge-target owner: only a resolved gateway-managed oauth2 - target (interactive or M2M) yields a per-server challenge; delegate-auth oauth2 - (whose keyless flow is upstream PKCE via the relay), every client-forwarded auth - type, OBO, api_key, unknown names, and CSV paths yield None (LIT-4864).""" + """Gateway challenges exclude unresolved, delegated, and client-forwarded targets.""" from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( _gateway_dcr_challenge_target, ) @@ -7333,7 +7335,11 @@ class TestAggregateGatewayDcrChallenge: (_server(MCPAuth.true_passthrough), None), (_server(MCPAuth.oauth_delegate), None), (_server(MCPAuth.oauth_delegate, dcr_bridge=True), None), - (_server(MCPAuth.api_key), None), + (_server(MCPAuth.api_key), "srv"), + (_server(MCPAuth.none, extra_headers=["Authorization"]), None), + (_server(None, extra_headers=["X-API-Key"]), None), + (_server(MCPAuth.none, extra_headers=["Authorization"], oauth_passthrough=True), None), + (_server(MCPAuth.oauth2_id_jag), None), (None, None), ] for resolved, expected in cases: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 763200c3709..a7a65bfe466 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -7651,12 +7651,6 @@ async def test_token_endpoint_client_secret_basic_without_secret_returns_400(): assert exc_info.value.status_code == 400 -# ------------------------------------------------------------------- -# Non-oauth2 (auth_type=none, access-group gated) servers must not be -# driven through the gateway OAuth authorize/token/register/discovery -# flow, and must not be advertised as OAuth-protected in discovery docs. -# ------------------------------------------------------------------- - def _access_group_none_server(server_name="access_group_server"): """A non-oauth2, access-group gated MCP server: no client_id, no OAuth.""" @@ -7794,35 +7788,38 @@ async def test_register_client_rejects_non_oauth2_server(): @pytest.mark.asyncio -async def test_oauth_protected_resource_404_for_non_oauth2_server(): - """Discovery must not advertise a none-auth server as an OAuth-protected resource.""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - _build_oauth_protected_resource_response, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - except ImportError: - pytest.skip("MCP discoverable endpoints not available") +@pytest.mark.parametrize( + "auth_type", (None, "none", "api_key", "bearer_token", "basic", "aws_sigv4", "authorization", "token") +) +@pytest.mark.parametrize("use_standard_pattern", (False, True)) +async def test_oauth_protected_resource_for_gateway_owned_auth(auth_type, use_standard_pattern): + from starlette.requests import Request + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_oauth_protected_resource_response, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + server = _access_group_none_server().model_copy(update={"auth_type": auth_type}) + request = Request( + {"type": "http", "scheme": "https", "path": "/", "headers": [(b"host", b"litellm.example.com")]} + ) global_mcp_server_manager.registry.clear() - server = _access_group_none_server() global_mcp_server_manager.registry[server.server_id] = server - - mock_request = MagicMock() - mock_request.base_url = "https://litellm.example.com/" - mock_request.headers = {} - try: - with pytest.raises(HTTPException) as exc_info: - await _build_oauth_protected_resource_response( - request=mock_request, - mcp_server_name="access_group_server", - use_standard_pattern=False, - ) - assert exc_info.value.status_code == 404 - assert "not an OAuth-protected resource" in str(exc_info.value.detail) + response = await _build_oauth_protected_resource_response( + request=request, + mcp_server_name="access_group_server", + use_standard_pattern=use_standard_pattern, + ) + resource_path = "/mcp/access_group_server" if use_standard_pattern else "/access_group_server/mcp" + assert response == { + "resource": f"https://litellm.example.com{resource_path}", + "authorization_servers": ["https://litellm.example.com/mcp"], + "scopes_supported": [], + } finally: global_mcp_server_manager.registry.clear() @@ -7914,9 +7911,7 @@ async def test_oauth_protected_resource_passthrough_none_auth_not_404(): @pytest.mark.asyncio async def test_oauth_protected_resource_404_for_unknown_server_name(): - """A discovery request for an unknown server name returns the same 404 as a non-oauth2 - server (not a 200 metadata doc with broken URLs), so the well-known paths cannot be used - to enumerate non-OAuth server names.""" + """Unknown server names must not produce metadata advertising nonexistent resources.""" try: from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( _build_oauth_protected_resource_response, From a9cce4f1ed15af0082a66233732d8e27f95b6bc8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:38:51 -0700 Subject: [PATCH 215/411] fix(guardrails): scan and rewrite Responses custom_tool_call output items Post-call guardrails on /v1/responses only treated function_call output items as tool calls, so a custom_tool_call item (Codex's exec shell tool on GPT-5.6 models) was never scanned or masked, non-streaming and streaming alike. Both item types now flow through the shared tool_call_dict_from_output_item helper, ended-stream delivery syncs the custom_tool_call_input delta/done events and the item's input field, and the completed-response scan key fingerprints both kinds of item. Non-streaming Responses tool-call MASK rewrites were also never written back to the output item even for function_call; they are now. --- .../transformation.py | 8 +- .../guardrail_translation/handler.py | 157 +++++++----- ...test_openai_responses_guardrail_handler.py | 229 +++++++++++++++++- 3 files changed, 322 insertions(+), 72 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 4fe069b0b7d..aa8337b6300 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -227,7 +227,7 @@ class _ChatToolCallDict(ChatCompletionToolCallChunk, total=False): provider_specific_fields: Mapping[str, object] -def _tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _ChatToolCallDict: +def tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _ChatToolCallDict: """Convert a ``function_call`` or ``custom_tool_call`` output item dict to a chat completions tool_call dict. Custom (grammar/freeform) tool calls carry their raw string payload in ``input`` rather than ``arguments``; both map to @@ -755,7 +755,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Tool calls accumulate into the single trailing tool_calls choice # like the typed branches above; a choice per call would hide every # call after choices[0] from chat clients - accumulated_tool_calls.append(_tool_call_dict_from_output_item(raw_item, tool_call_index)) + accumulated_tool_calls.append(tool_call_dict_from_output_item(raw_item, tool_call_index)) tool_call_index += 1 elif handle_raw_dict_callback is not None: choice, index = handle_raw_dict_callback(item=raw_item, index=index) @@ -1409,7 +1409,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # New output item added output_item = parsed_chunk.get("item", {}) if output_item.get("type") in ("function_call", "custom_tool_call"): - converted: Final = _tool_call_dict_from_output_item(output_item, parsed_chunk.get("output_index", 0)) + converted: Final = tool_call_dict_from_output_item(output_item, parsed_chunk.get("output_index", 0)) provider_specific_fields: Final = converted.get("provider_specific_fields") function_chunk: Final = ChatCompletionToolCallFunctionChunk( @@ -1484,7 +1484,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): index=0, delta=Delta( tool_calls=( - _tool_call_dict_from_output_item( + tool_call_dict_from_output_item( output_item, parsed_chunk.get("output_index", 0) ), ) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 280a8670c36..e78e1f56915 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -37,7 +37,6 @@ from itertools import accumulate, chain, repeat from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast -from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from pydantic import BaseModel, TypeAdapter from typing_extensions import ReadOnly, TypedDict @@ -45,6 +44,7 @@ from litellm._logging import verbose_proxy_logger from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, OpenAiResponsesToChatCompletionStreamIterator, + tool_call_dict_from_output_item, ) from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, @@ -84,7 +84,6 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import ( GenericResponseOutputItem, - OutputFunctionToolCall, OutputText, ) from litellm.types.utils import GenericGuardrailAPIInputs @@ -140,8 +139,18 @@ _TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset( ) -_FUNCTION_CALL_ARGUMENT_EVENT_TYPES: Final = frozenset( - {"response.function_call_arguments.delta", "response.function_call_arguments.done"} +_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call"}) +_TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"function_call": "arguments", "custom_tool_call": "input"} +) +_TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES: Final = frozenset( + {"response.function_call_arguments.delta", "response.custom_tool_call_input.delta"} +) +_TOOL_CALL_PAYLOAD_DONE_EVENT_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"response.function_call_arguments.done": "arguments", "response.custom_tool_call_input.done": "input"} +) +_TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | frozenset( + _TOOL_CALL_PAYLOAD_DONE_EVENT_FIELDS ) _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( @@ -180,8 +189,20 @@ def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapp return {**item, field: converted_value} # mutable-ok: request input items must stay JSON-plain dicts -def _is_function_call_item(item: object) -> bool: - return isinstance(item, Mapping) and item.get("type") in ("function_call", "custom_tool_call") +def _is_tool_call_item(item: object) -> bool: + return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES + + +def _tool_call_output_item_mapping(item: object) -> Mapping[str, Any] | None: + if stream_item_field(item, "type") not in _TOOL_CALL_ITEM_TYPES: + return None + if isinstance(item, Mapping): + return cast("Mapping[str, Any]", item) # cast-ok: output items are str-keyed JSON objects + return item.model_dump() if isinstance(item, BaseModel) else None + + +def _is_tool_call_output_item(item: object) -> bool: + return _tool_call_output_item_mapping(item) is not None def _last_message_role(messages: Sequence[object]) -> str | None: @@ -205,7 +226,7 @@ def _provenance_unit_bounds( start_indexes: Final = tuple( index for index in range(len(raw_input)) - if index == 0 or not (_is_function_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant") + if index == 0 or not (_is_tool_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant") ) return tuple(zip(start_indexes, (*start_indexes[1:], len(raw_input)))) @@ -603,7 +624,7 @@ class OpenAIResponsesHandler(BaseTranslation): - response.output is a list of output items - Each output item can be: * GenericResponseOutputItem with a content list of OutputText objects - * ResponseFunctionToolCall with tool call data + * ResponseFunctionToolCall or CustomToolCallOutputItem with tool call data - Each OutputText object has a text field """ @@ -668,6 +689,7 @@ class OpenAIResponsesHandler(BaseTranslation): if response_model: inputs["model"] = response_model + pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_to_check) guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, request_data=request_data, @@ -676,6 +698,7 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") # Step 3: Map guardrail responses back to original response structure await self._apply_guardrail_responses_to_output( @@ -683,6 +706,15 @@ class OpenAIResponsesHandler(BaseTranslation): responses=guardrailed_texts, task_mappings=task_mappings, ) + self._write_tool_call_rewrites_to_output( + tool_call_items=tuple(item for item in response_output if _is_tool_call_output_item(item)), + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + post_guardrail_tool_calls=_tool_call_shapes( + returned_tool_calls + if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) + else tool_calls_to_check + ), + ) verbose_proxy_logger.debug("OpenAI Responses API: Processed output response: %s", response) @@ -933,11 +965,12 @@ class OpenAIResponsesHandler(BaseTranslation): guardrail_name: str, ) -> None: """Write ended-stream guardrail tool-call rewrites into the completed - envelope's ``function_call`` items and sync the earlier stream events, - keyed by ``call_id``. The guardrail sees the envelope's function calls - in output order, which is how a rewritten call finds its ``call_id``; - the stream events find their call through the ``call_id`` on - ``output_item`` events and the ``item_id`` on argument events, since an + envelope's ``function_call`` and ``custom_tool_call`` items and sync the + earlier stream events, keyed by ``call_id``. The guardrail sees the + envelope's tool calls in output order, which is how a rewritten call + finds its ``call_id``; the stream events find their call through the + ``call_id`` on ``output_item`` events and the ``item_id`` on argument + and custom-input events, since an event's ``output_index`` need not match the envelope's (the chat bridge numbers tool calls from 1 while the envelope lists them after the message). A rewrite whose calls do not line up with the envelope, or @@ -945,18 +978,16 @@ class OpenAIResponsesHandler(BaseTranslation): pipeline executor discards it and releases the original events.""" if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - function_call_items: Final = tuple( - output_item for output_item in outputs if stream_item_field(output_item, "type") == "function_call" - ) + tool_call_items: Final = tuple(output_item for output_item in outputs if _is_tool_call_output_item(output_item)) call_ids: Final = tuple( call_id - for output_item in function_call_items + for output_item in tool_call_items if isinstance(call_id := stream_item_field(output_item, "call_id"), str) and call_id ) stream_events: Final = responses_so_far[:-1] - call_id_by_item_id: Final = self._function_call_ids_by_item_id(stream_events) + call_id_by_item_id: Final = self._tool_call_ids_by_item_id(stream_events) event_call_ids: Final = tuple( - self._function_call_event_call_id(event, call_id_by_item_id) for event in stream_events + self._tool_call_event_call_id(event, call_id_by_item_id) for event in stream_events ) rewrites_by_call_id: Final = MappingProxyType( { @@ -966,11 +997,11 @@ class OpenAIResponsesHandler(BaseTranslation): } ) unresolved_argument_event: Final = any( - call_id is None and stream_item_field(event, "type") in _FUNCTION_CALL_ARGUMENT_EVENT_TYPES + call_id is None and stream_item_field(event, "type") in _TOOL_CALL_PAYLOAD_EVENT_TYPES for event, call_id in zip(stream_events, event_call_ids) ) if ( - len(call_ids) != len(function_call_items) + len(call_ids) != len(tool_call_items) or len(frozenset(call_ids)) != len(call_ids) or len(call_ids) != len(post_guardrail_tool_calls) or unresolved_argument_event @@ -981,10 +1012,10 @@ class OpenAIResponsesHandler(BaseTranslation): raise UndeliverableStreamRewrite(guardrail_name) for output_item, rewrite in ( (output_item, rewrites_by_call_id[call_id]) - for output_item, call_id in zip(function_call_items, call_ids) + for output_item, call_id in zip(tool_call_items, call_ids) if call_id in rewrites_by_call_id ): - self._write_function_call_item(output_item, rewrite.name, rewrite.arguments) + self._write_tool_call_item(output_item, rewrite.name, rewrite.arguments) delta_replacements: Final = MappingProxyType( {call_id: chain((rewrite.arguments,), repeat("")) for call_id, rewrite in rewrites_by_call_id.items()} ) @@ -992,16 +1023,18 @@ class OpenAIResponsesHandler(BaseTranslation): if call_id not in rewrites_by_call_id: continue match stream_item_field(event, "type"): - case "response.function_call_arguments.delta": + case str() as event_type if event_type in _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES: self._write_event_field(event, "delta", next(delta_replacements[call_id])) - case "response.function_call_arguments.done": - self._write_event_field(event, "arguments", rewrites_by_call_id[call_id].arguments) + case str() as event_type if event_type in _TOOL_CALL_PAYLOAD_DONE_EVENT_FIELDS: + self._write_event_field( + event, _TOOL_CALL_PAYLOAD_DONE_EVENT_FIELDS[event_type], rewrites_by_call_id[call_id].arguments + ) case "response.output_item.added": - self._write_function_call_item( + self._write_tool_call_item( stream_item_field(event, "item"), rewrites_by_call_id[call_id].name, None ) case "response.output_item.done": - self._write_function_call_item( + self._write_tool_call_item( stream_item_field(event, "item"), rewrites_by_call_id[call_id].name, rewrites_by_call_id[call_id].arguments, @@ -1009,8 +1042,23 @@ class OpenAIResponsesHandler(BaseTranslation): case _: pass + def _write_tool_call_rewrites_to_output( + self, + tool_call_items: Sequence[object], + pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], + post_guardrail_tool_calls: tuple[_ToolCallShape, ...], + ) -> None: + if len(tool_call_items) != len(post_guardrail_tool_calls): + return + for output_item, after in ( + (output_item, after) + for output_item, before, after in zip(tool_call_items, pre_guardrail_tool_calls, post_guardrail_tool_calls) + if after != before + ): + self._write_tool_call_item(output_item, after.name, after.arguments) + @staticmethod - def _function_call_ids_by_item_id(stream_events: Sequence[object]) -> Mapping[str, str]: + def _tool_call_ids_by_item_id(stream_events: Sequence[object]) -> Mapping[str, str]: items: Final = tuple( stream_item_field(event, "item") for event in stream_events @@ -1020,32 +1068,35 @@ class OpenAIResponsesHandler(BaseTranslation): { item_id: call_id for item in items - if stream_item_field(item, "type") == "function_call" + if stream_item_field(item, "type") in _TOOL_CALL_ITEM_TYPES and isinstance(item_id := stream_item_field(item, "id"), str) and isinstance(call_id := stream_item_field(item, "call_id"), str) } ) @staticmethod - def _function_call_event_call_id(event: object, call_id_by_item_id: Mapping[str, str]) -> str | None: + def _tool_call_event_call_id(event: object, call_id_by_item_id: Mapping[str, str]) -> str | None: event_type: Final = stream_item_field(event, "type") - if event_type in _FUNCTION_CALL_ARGUMENT_EVENT_TYPES: + if event_type in _TOOL_CALL_PAYLOAD_EVENT_TYPES: item_id: Final = stream_item_field(event, "item_id") return call_id_by_item_id.get(item_id) if isinstance(item_id, str) else None if event_type not in _OUTPUT_ITEM_EVENT_TYPES: return None item: Final = stream_item_field(event, "item") call_id: Final = stream_item_field(item, "call_id") - return call_id if stream_item_field(item, "type") == "function_call" and isinstance(call_id, str) else None + return ( + call_id if stream_item_field(item, "type") in _TOOL_CALL_ITEM_TYPES and isinstance(call_id, str) else None + ) @staticmethod - def _write_function_call_item(item: object, name: str | None, arguments: str | None) -> None: + def _write_tool_call_item(item: object, name: str | None, payload: str | None) -> None: if item is None: return if name is not None: OpenAIResponsesHandler._write_event_field(item, "name", name) - if arguments is not None: - OpenAIResponsesHandler._write_event_field(item, "arguments", arguments) + item_type: Final = stream_item_field(item, "type") + if payload is not None and isinstance(item_type, str) and item_type in _TOOL_CALL_PAYLOAD_FIELDS: + OpenAIResponsesHandler._write_event_field(item, _TOOL_CALL_PAYLOAD_FIELDS[item_type], payload) def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool: """ @@ -1073,7 +1124,7 @@ class OpenAIResponsesHandler(BaseTranslation): def _completed_response_scan_key(response: object) -> StreamingScanKey: output_items: Final = stream_item_items(response, "output") message_items: Final = tuple( - item for item in output_items if stream_item_field(item, "type") != "function_call" + item for item in output_items if stream_item_field(item, "type") not in _TOOL_CALL_ITEM_TYPES ) return StreamingScanKey( texts=tuple( @@ -1085,7 +1136,7 @@ class OpenAIResponsesHandler(BaseTranslation): tool_calls=tuple( stream_item_fingerprint(item) for item in output_items - if stream_item_field(item, "type") == "function_call" + if stream_item_field(item, "type") in _TOOL_CALL_ITEM_TYPES ), stream_ended=True, ) @@ -1196,34 +1247,10 @@ class OpenAIResponsesHandler(BaseTranslation): Override this method to customize text/image/tool extraction logic. """ - # Check if this is a tool call (OutputFunctionToolCall) - if isinstance(output_item, OutputFunctionToolCall) or ( - isinstance(output_item, BaseModel) - and hasattr(output_item, "type") - and getattr(output_item, "type") == "function_call" - ): + tool_call_item: Final = _tool_call_output_item_mapping(output_item) + if tool_call_item is not None: if tool_calls_to_check is not None: - tool_call_dict = ( - LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call( - tool_call_item=output_item, - index=output_idx, - ) - ) - tool_calls_to_check.append(cast(ChatCompletionToolCallChunk, tool_call_dict)) - return - elif isinstance(output_item, dict) and output_item.get("type") == "function_call": - # Handle dict representation of tool call - if tool_calls_to_check is not None: - # Convert dict to ResponseFunctionToolCall for processing - try: - tool_call_obj: Final = ResponseFunctionToolCall(**output_item) - tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call( - tool_call_item=tool_call_obj, - index=output_idx, - ) - tool_calls_to_check.append(cast(ChatCompletionToolCallChunk, tool_call_dict)) - except Exception: - pass + tool_calls_to_check.append(tool_call_dict_from_output_item(tool_call_item, output_idx)) return # Handle both GenericResponseOutputItem and dict diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 6ed4ec6618f..628ada05267 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -14,7 +14,12 @@ import pytest from fastapi import HTTPException -from openai.types.responses import ResponseFunctionToolCall +from openai.types.responses import ( + ResponseCustomToolCall, + ResponseCustomToolCallInputDeltaEvent, + ResponseCustomToolCallInputDoneEvent, + ResponseFunctionToolCall, +) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -27,7 +32,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.responses.main import GenericResponseOutputItem, OutputText +from litellm.types.responses.main import CustomToolCallOutputItem, GenericResponseOutputItem, OutputText from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs @@ -57,6 +62,37 @@ class MockGuardrail(CustomGuardrail): return inputs +class PersimmonMaskingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + tool_calls = [ + { + **tool_call, + "function": { + **tool_call["function"], + "arguments": tool_call["function"]["arguments"].replace("persimmon", "[MASKED]"), + }, + } + for tool_call in inputs.get("tool_calls", []) + ] + return {**inputs, "tool_calls": tool_calls} + + +CUSTOM_TOOL_CALL_ITEM = { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_exec_1", + "name": "exec", + "input": "echo persimmon", + "status": "completed", +} + + class TestOpenAIResponsesHandlerDiscovery: """Test that the handler is properly discovered by the guardrail system""" @@ -628,6 +664,77 @@ class TestOpenAIResponsesHandlerToolCallExtraction: == '{"location":"Boston, MA","unit":"celsius"}' ) + @pytest.mark.parametrize( + "output_item", + [ + dict(CUSTOM_TOOL_CALL_ITEM), + CustomToolCallOutputItem(**CUSTOM_TOOL_CALL_ITEM), + ResponseCustomToolCall(**{key: value for key, value in CUSTOM_TOOL_CALL_ITEM.items() if key != "status"}), + ], + ids=["dict", "litellm_typed", "openai_typed"], + ) + def test_extract_custom_tool_call_input_as_arguments(self, output_item): + handler = OpenAIResponsesHandler() + texts_to_check: List[str] = [] + tool_calls_to_check: List[Any] = [] + + handler._extract_output_text_and_images( + output_item=output_item, + output_idx=2, + texts_to_check=texts_to_check, + images_to_check=[], + task_mappings=[], + tool_calls_to_check=tool_calls_to_check, + ) + + assert texts_to_check == [] + assert tool_calls_to_check == [ + { + "id": "call_exec_1", + "type": "function", + "function": {"name": "exec", "arguments": "echo persimmon"}, + "index": 2, + } + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize("typed", [False, True], ids=["dict", "typed"]) + async def test_process_output_response_writes_tool_call_rewrites_back(self, typed): + handler = OpenAIResponsesHandler() + function_call = { + "type": "function_call", + "id": "fc_1", + "call_id": "call_fn_1", + "name": "lookup_fruit", + "arguments": '{"fruit": "persimmon"}', + "status": "completed", + } + message = { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "running persimmon", "annotations": []}], + } + payload = { + "id": "resp_1", + "created_at": 1, + "model": "gpt-5.6", + "object": "response", + "status": "completed", + "output": [message, function_call, dict(CUSTOM_TOOL_CALL_ITEM)], + } + response = ResponsesAPIResponse.model_validate(payload) if typed else payload + + result = await handler.process_output_response(response, PersimmonMaskingGuardrail(guardrail_name="mask")) + + output = result.output if typed else result["output"] + function_item, custom_item = output[1], output[2] + assert (function_item.arguments if typed else function_item["arguments"]) == '{"fruit": "[MASKED]"}' + assert (custom_item.input if typed else custom_item["input"]) == "echo [MASKED]" + assert (custom_item.name if typed else custom_item["name"]) == "exec" + assert (output[0].content[0].text if typed else output[0]["content"][0]["text"]) == "running persimmon" + @pytest.mark.asyncio async def test_process_output_response_with_tool_calls(self): """Test processing output response containing function tool calls""" @@ -1315,6 +1422,109 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert completed_event.response.output[0].arguments == '{"fruit": "[MASKED]"}' assert completed_event.response.output[0].name == "lookup_fruit" + @staticmethod + def _ended_custom_tool_call_stream_events() -> List[dict]: + def item(input_text: str, status: str) -> dict: + return {**CUSTOM_TOOL_CALL_ITEM, "input": input_text, "status": status} + + return [ + {"type": "response.output_item.added", "output_index": 0, "item": item("", "in_progress")}, + {"type": "response.custom_tool_call_input.delta", "item_id": "ctc_1", "output_index": 0, "delta": "echo "}, + {"type": "response.custom_tool_call_input.delta", "item_id": "ctc_1", "output_index": 0, "delta": "persimmon"}, + {"type": "response.custom_tool_call_input.done", "item_id": "ctc_1", "output_index": 0, "input": "echo persimmon"}, + {"type": "response.output_item.done", "output_index": 0, "item": item("echo persimmon", "completed")}, + { + "type": "response.completed", + "response": { + "id": "resp_123", + "created_at": 1, + "model": "gpt-5.6", + "output": [item("echo persimmon", "completed")], + "status": "completed", + }, + }, + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_custom_tool_call_events(self): + handler = OpenAIResponsesHandler() + events = self._ended_custom_tool_call_stream_events() + + result = await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=PersimmonMaskingGuardrail(guardrail_name="mask"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert result is events + assert events[0]["item"]["input"] == "" + assert events[1]["delta"] == "echo [MASKED]" + assert events[2]["delta"] == "" + assert events[3]["input"] == "echo [MASKED]" + assert events[4]["item"]["input"] == "echo [MASKED]" + assert events[5]["response"]["output"][0]["input"] == "echo [MASKED]" + assert events[5]["response"]["output"][0]["name"] == "exec" + assert "arguments" not in events[5]["response"]["output"][0] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_typed_custom_tool_call_events(self): + from litellm.types.llms.openai import ( + OutputItemAddedEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ) + + handler = OpenAIResponsesHandler() + typed_events: List[Any] = [ + model.model_validate({**event, "sequence_number": sequence_number}) + for sequence_number, (model, event) in enumerate( + zip( + ( + OutputItemAddedEvent, + ResponseCustomToolCallInputDeltaEvent, + ResponseCustomToolCallInputDeltaEvent, + ResponseCustomToolCallInputDoneEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ), + self._ended_custom_tool_call_stream_events(), + ) + ) + ] + completed_event = typed_events[5] + assert isinstance(completed_event.response.output[0], CustomToolCallOutputItem) + + await handler.process_output_streaming_response( + responses_so_far=typed_events, + guardrail_to_apply=PersimmonMaskingGuardrail(guardrail_name="mask"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert typed_events[1].delta == "echo [MASKED]" + assert typed_events[2].delta == "" + assert typed_events[3].input == "echo [MASKED]" + assert typed_events[4].item.input == "echo [MASKED]" + assert completed_event.response.output[0].input == "echo [MASKED]" + assert completed_event.response.output[0].name == "exec" + + @pytest.mark.asyncio + async def test_deliver_ended_stream_custom_tool_call_rewrite_without_matching_events_fails_closed(self): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = OpenAIResponsesHandler() + events = self._ended_custom_tool_call_stream_events() + events[5]["response"]["output"] = [{**events[5]["response"]["output"][0], "call_id": "call_999"}] + + with pytest.raises(UndeliverableStreamRewrite): + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=PersimmonMaskingGuardrail(guardrail_name="mask"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + @staticmethod def _bridged_function_call_stream_events() -> List[dict]: reasoning = {"type": "reasoning", "id": "rs_1", "summary": []} @@ -2747,8 +2957,21 @@ class TestOpenAIResponsesHandlerStreamingScanKey: assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] assert ended_key != open_key + def test_completed_event_with_a_custom_tool_call_changes_the_key(self): + handler = OpenAIResponsesHandler() + message = {"type": "message", "content": [{"type": "output_text", "text": "hi"}]} + ended_key = handler.get_streaming_scan_key( + [self._delta(0, "hi"), self._completed(1, [message, dict(CUSTOM_TOOL_CALL_ITEM)])] + ) + rewritten_key = handler.get_streaming_scan_key( + [self._delta(0, "hi"), self._completed(1, [message, {**CUSTOM_TOOL_CALL_ITEM, "input": "echo kumquat"}])] + ) + assert ended_key.texts == ("hi",) + assert len(ended_key.tool_calls) == 1 and "echo persimmon" in ended_key.tool_calls[0] + assert rewritten_key != ended_key + def test_completed_event_reads_every_output_text_part(self): - from litellm.types.responses.main import GenericResponseOutputItem, OutputText + from litellm.types.responses.main import CustomToolCallOutputItem, GenericResponseOutputItem, OutputText item = GenericResponseOutputItem( type="message", From feeda1a36ce9f4155e443bbb264950ac0139f7a0 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:32:06 -0700 Subject: [PATCH 216/411] fix(mcp): report resolved upstream authentication in debug headers --- .../_experimental/mcp_server/mcp_debug.py | 161 ++++++++++++------ .../mcp_server/mcp_server_manager.py | 34 +++- .../outbound_credentials/resolver.py | 31 ++++ .../mcp_server/outbound_credentials/types.py | 26 ++- .../proxy/_experimental/mcp_server/server.py | 14 +- .../outbound_credentials/test_resolver.py | 66 +++++++ .../mcp_server/test_mcp_debug.py | 161 ++++++++++-------- .../mcp_server/test_mcp_server_manager.py | 106 ++++++++++++ 8 files changed, 470 insertions(+), 129 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_debug.py b/litellm/proxy/_experimental/mcp_server/mcp_debug.py index 6aaa00ae415..b25ed27becc 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_debug.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_debug.py @@ -22,7 +22,16 @@ Response headers returned (all values are masked for safety): x-mcp-debug-auth-resolution Which auth priority was used for the outbound MCP call: ``per-request-header``, ``m2m-client-credentials``, ``static-token``, - ``oauth2-passthrough``, or ``no-auth``. + ``oauth2-passthrough``, ``stored-user-token``, ``token-exchange``, + ``id-jag``, ``aws-sigv4``, ``extra-headers``, or ``no-auth``. + ``unresolved`` means no outcome was available before the first response + frame; ``multiple`` means several servers resolved credentials; + ``not-applicable`` covers stdio; ``resolution-failed`` is a resolver error. + + x-mcp-debug-auth-resolutions + For multiple servers, a JSON map of server IDs to resolution labels. + At most 32 entries are included; x-mcp-debug-auth-resolutions-truncated + is true when additional servers were omitted. No credentials are included. x-mcp-debug-outbound-url The upstream MCP server URL that will receive the request. @@ -58,10 +67,16 @@ header is free for OAuth2 discovery:: Symptom: ``x-mcp-debug-oauth2-token`` shows ``(none)`` and ``x-mcp-debug-auth-resolution`` shows ``no-auth``. -This means the client didn't go through the OAuth2 flow. Check that: -1. The ``Authorization`` header is NOT set as a static header in the client config. -2. The ``.well-known/oauth-protected-resource`` endpoint returns valid metadata. -3. The MCP server in LiteLLM config has ``auth_type: oauth2``. +``no-auth`` means the resolved upstream client carries no authentication. +An absent inbound OAuth2 token does not imply the user skipped OAuth: the gateway +can retrieve a stored per-user token, reported as ``stored-user-token``. +``unresolved`` is used when a stream starts before credential resolution, or a +request (such as initialization or a cached tool listing) resolves no credential. +Debug reporting does not fetch credentials or delay a streaming frame to resolve them. +``extra-headers`` identifies supplied headers that won over the resolver or were +the only headers supplied; their values are never inspected to guess a scheme. +``per-request-header`` denotes a legacy credential override, including a BYOK +credential supplied by the gateway; it does not imply a caller-supplied token. **Common issue: M2M token used instead of user token** @@ -69,8 +84,8 @@ Symptom: ``x-mcp-debug-auth-resolution`` shows ``m2m-client-credentials``. This means the server has ``client_id``/``client_secret``/``token_url`` configured and LiteLLM is fetching a machine-to-machine token instead of -using the per-user OAuth2 token. If you want per-user tokens, remove the -client credentials from the server config. +using the per-user OAuth2 token. For gateway-stored per-user tokens, +configure ``oauth2_flow: authorization_code``. Usage from Claude Code:: @@ -85,14 +100,16 @@ Usage with curl:: http://localhost:4000/mcp/atlassian_mcp """ -from typing import TYPE_CHECKING, Final +import json +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import Final +from starlette.requests import HTTPConnection from starlette.types import Message, Send from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker - -if TYPE_CHECKING: - from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution # Header the client sends to opt into debug mode MCP_DEBUG_REQUEST_HEADER: Final = "x-litellm-mcp-debug" @@ -101,6 +118,83 @@ MCP_DEBUG_REQUEST_HEADER: Final = "x-litellm-mcp-debug" _RESPONSE_HEADER_PREFIX: Final = "x-mcp-debug" +MCP_AUTH_DIAGNOSTICS_SCOPE_KEY: Final = "litellm.mcp.auth_diagnostics" + + +def record_auth_resolution(server_id: str, source: AuthResolution) -> None: + from mcp.server.lowlevel.server import request_ctx + + context: Final[object] = request_ctx.get(None) + request: Final[object] = getattr(context, "request", None) + if isinstance(request, HTTPConnection): + diagnostics: Final[object] = request.scope.get(MCP_AUTH_DIAGNOSTICS_SCOPE_KEY) + if isinstance(diagnostics, MCPAuthDiagnostics): + diagnostics.record(server_id, source) + + +class MCPAuthDiagnostics: + def __init__(self) -> None: + self._outcomes: tuple[tuple[str, AuthResolution], ...] = () + + def record(self, server_id: str, resolution: AuthResolution) -> None: + self._outcomes = tuple(item for item in self._outcomes if item[0] != server_id) + ((server_id, resolution),) + + def resolution(self) -> str: + match self._outcomes: + case (): + return AuthResolution.unresolved.value + case ((_, source),): + return source.value + case _: + return AuthResolution.multiple.value + + def headers(self) -> Mapping[str, str]: + if len(self._outcomes) <= 1: + return MappingProxyType({"x-mcp-debug-auth-resolution": self.resolution()}) + return MappingProxyType( + { + "x-mcp-debug-auth-resolution": AuthResolution.multiple.value, + "x-mcp-debug-auth-resolutions": json.dumps( + { + server_id: source.value for server_id, source in self._outcomes[:32] + }, # mutable-ok: JSON encoder requires a concrete dict + separators=(",", ":"), + ensure_ascii=True, + ), + **( + MappingProxyType({"x-mcp-debug-auth-resolutions-truncated": "true"}) + if len(self._outcomes) > 32 + else MappingProxyType({}) + ), + } + ) + + +class _DiagnosticSend: + def __init__(self, send: Send, headers: Mapping[str, str], resolution: Callable[[], Mapping[str, str]]) -> None: + self._send = send + self._headers = headers + self._resolution = resolution + self._start: Message | None = None + + async def __call__(self, message: Message) -> None: + if message["type"] == "http.response.start": + self._start = message + return + if self._start is not None: + start: Final = self._start + self._start = None + headers: Final = MappingProxyType({**self._headers, **self._resolution()}) + await self._send( + { # mutable-ok: ASGI send consumes a mutable message mapping + **start, + "headers": tuple(start.get("headers", ())) + + tuple((key.encode(), value.encode()) for key, value in headers.items()), + } + ) + await self._send(message) + + class MCPDebug: """ Static helper class for MCP OAuth2 debug headers. @@ -144,37 +238,6 @@ class MCPDebug: return val.strip().lower() in ("true", "1", "yes") return False - @staticmethod - def resolve_auth_resolution( - server: "MCPServer", - mcp_auth_header: str | None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None, - oauth2_headers: dict[str, str] | None, - ) -> str: - """ - Determine which auth priority will be used for the outbound MCP call. - - Returns one of: ``per-request-header``, ``m2m-client-credentials``, - ``static-token``, ``oauth2-passthrough``, or ``no-auth``. - """ - from litellm.types.mcp import MCPAuth - - has_server_specific: Final = bool( - mcp_server_auth_headers - and ( - mcp_server_auth_headers.get(server.alias or "") or mcp_server_auth_headers.get(server.server_name or "") - ) - ) - if has_server_specific or mcp_auth_header: - return "per-request-header" - if server.has_client_credentials: - return "m2m-client-credentials" - if server.authentication_token: - return "static-token" - if oauth2_headers and server.auth_type == MCPAuth.oauth2: - return "oauth2-passthrough" - return "no-auth" - @staticmethod def build_debug_headers( *, @@ -244,12 +307,17 @@ class MCPDebug: return debug @staticmethod - def wrap_send_with_debug_headers(send: Send, debug_headers: dict[str, str]) -> Send: + def wrap_send_with_debug_headers( + send: Send, debug_headers: Mapping[str, str], resolution: Callable[[], Mapping[str, str]] | None = None + ) -> Send: """ Return a new ASGI ``send`` callable that injects *debug_headers* into the ``http.response.start`` message. """ + if resolution is not None: + return _DiagnosticSend(send, debug_headers, resolution) + async def _send_with_debug(message: Message) -> None: if message["type"] == "http.response.start": headers: Final = list(message.get("headers", [])) @@ -266,8 +334,6 @@ class MCPDebug: raw_headers: dict[str, str] | None, scope: dict, mcp_servers: list[str] | None, - mcp_auth_header: str | None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None, oauth2_headers: dict[str, str] | None, client_ip: str | None, ) -> dict[str, str]: @@ -288,16 +354,13 @@ class MCPDebug: server_url: str | None = None server_auth_type: str | None = None - auth_resolution = "no-auth" + auth_resolution: Final = AuthResolution.unresolved.value for server_name in mcp_servers or []: server = global_mcp_server_manager.get_mcp_server_by_name(server_name, client_ip=client_ip) if server: server_url = server.url server_auth_type = server.auth_type - auth_resolution = MCPDebug.resolve_auth_resolution( - server, mcp_auth_header, mcp_server_auth_headers, oauth2_headers - ) break scope_headers: Final = MCPRequestHandler._safe_get_headers_from_scope(scope) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index d7f238f142c..116bcc715d4 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -80,6 +80,7 @@ from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( raise_classified_list_failure, upstream_auth_challenge, ) +from litellm.proxy._experimental.mcp_server.mcp_debug import record_auth_resolution from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( MCPPerUserTokenCache, mcp_per_user_token_cache, @@ -108,12 +109,14 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_sto from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import ( LazyPerUserOAuthTokenStore, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_provider import ( build_token_exchanger, ) from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( DEFAULT_CREDENTIAL_HEADER, AuthorizationCodeConfig, + AuthResolution, ClientCredentialsConfig, CredError, IdJagConfig, @@ -3832,13 +3835,21 @@ class MCPServerManager: (authorization_code's browser-OAuth 401, token_exchange's RFC 9728 challenge) or maps any other ``CredError`` onto its public HTTP status; it never returns an error as a value. """ - match await provider.resolve_credentials(to_subject(user_api_key_auth, subject_token), spec): - case Ok(auth): + match await resolve_credentials_with_source(provider, to_subject(user_api_key_auth, subject_token), spec): + case Ok(credential): + auth: Final = credential.auth # NoOpAuth has no header_name and so never conflicts. header_name: Final[str | None] = getattr(auth, "header_name", None) if header_name is None or not extra_headers: + source: Final = ( + AuthResolution.extra_headers + if credential.source == AuthResolution.no_auth and extra_headers + else credential.source + ) + record_auth_resolution(server.server_id, source) return auth, extra_headers if not has_header(extra_headers, header_name): + record_auth_resolution(server.server_id, credential.source) return auth, extra_headers if isinstance( spec.config, @@ -3853,11 +3864,14 @@ class MCPServerManager: # one-shot 401 refetch is lost with it). Drop only the header the resolved # credential is about to occupy, so a static credential the operator aimed at a # DIFFERENT header still reaches upstream. + record_auth_resolution(server.server_id, credential.source) return auth, without_header(extra_headers, header_name) # Other modes: an Authorization already supplied via extra_headers (a forwarded caller # header or static_headers) is intentional and wins; v1 applies those last. + record_auth_resolution(server.server_id, AuthResolution.extra_headers) return None, extra_headers case Error(err): + record_auth_resolution(server.server_id, AuthResolution.failed) if err.tag == "unauthorized" and isinstance(spec.config, AuthorizationCodeConfig): # authorization_code's missing per-user token -> the per-server browser-OAuth # challenge, built here where the full MCPServer is in hand. @@ -3960,6 +3974,7 @@ class MCPServerManager: Returns: Configured MCP client instance. """ + record_auth_resolution(server.server_id, AuthResolution.unresolved) resolved_server: Final = await self.ensure_oauth_metadata_discovered(server) transport: Final = resolved_server.transport or MCPTransport.sse spec = None if transport == MCPTransport.stdio else _to_server_spec_fail_closed(resolved_server) @@ -4032,6 +4047,7 @@ class MCPServerManager: env=resolved_env, ) + record_auth_resolution(server.server_id, AuthResolution.not_applicable) return MCPClient( server_url="", # Not used for stdio transport_type=transport, @@ -4086,6 +4102,20 @@ class MCPServerManager: aws_session_name=resolved_server.aws_session_name, ) + legacy_source: Final = ( + AuthResolution.aws_sigv4 + if aws_auth is not None + else AuthResolution.extra_headers + if extra_headers and has_header(extra_headers, auth_header_name or "Authorization") + else AuthResolution.per_request_header + if mcp_auth_header + else AuthResolution.static_token + if auth_value + else AuthResolution.extra_headers + if extra_headers + else AuthResolution.no_auth + ) + record_auth_resolution(server.server_id, legacy_source) return MCPClient( server_url=server_url, transport_type=transport, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py index 8328aae01ab..85c7f68719d 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py @@ -65,6 +65,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( ApiKeyConfig, AuthorizationCodeConfig, + AuthResolution, AuthSpecKind, AwsSigV4Config, Byok, @@ -76,6 +77,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( NoneConfig, PassthroughConfig, PrivateKeyJwtAuth, + ResolvedCredential, ServerSpec, SharedKey, Subject, @@ -448,3 +450,32 @@ def _client_auth_fingerprint(client_auth: ClientAuth) -> str: def _not_implemented(kind: AuthSpecKind) -> Result[httpx.Auth, CredError]: return Error(CredError.of_not_implemented(f"{kind.value}: resolver arm not implemented yet")) + + +async def resolve_credentials_with_source( + provider: UpstreamCredentialProvider, subject: Subject, server: ServerSpec +) -> Result[ResolvedCredential, CredError]: + match await provider.resolve_credentials(subject, server): + case Error(err): + return Error(err) + case Ok(auth): + if isinstance(auth, NoOpAuth): + return Ok(ResolvedCredential(auth, AuthResolution.no_auth)) + match server.config: + case NoneConfig(): + return Ok(ResolvedCredential(auth, AuthResolution.no_auth)) + case ApiKeyConfig(): + return Ok(ResolvedCredential(auth, AuthResolution.static_token)) + case PassthroughConfig(): + return Ok(ResolvedCredential(auth, AuthResolution.oauth2_passthrough)) + case ClientCredentialsConfig(): + return Ok(ResolvedCredential(auth, AuthResolution.client_credentials)) + case TokenExchangeConfig(): + return Ok(ResolvedCredential(auth, AuthResolution.token_exchange)) + case IdJagConfig(): + return Ok(ResolvedCredential(auth, AuthResolution.id_jag)) + case AuthorizationCodeConfig(): + return Ok(ResolvedCredential(auth, AuthResolution.stored_user_token)) + case AwsSigV4Config(): + return Ok(ResolvedCredential(auth, AuthResolution.aws_sigv4)) + assert_never(server.config) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py index 632dc57dcf6..d186724fd9f 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py @@ -26,10 +26,11 @@ union (see `result.py`), not `expression.Result`. from __future__ import annotations from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum from typing import Annotated, Final, Literal +import httpx from expression import case, tag, tagged_union from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator from typing_extensions import assert_never @@ -46,6 +47,29 @@ from litellm.types.mcp import ( ) +class AuthResolution(str, Enum): + no_auth = "no-auth" + stored_user_token = "stored-user-token" + static_token = "static-token" + per_request_header = "per-request-header" + oauth2_passthrough = "oauth2-passthrough" + client_credentials = "m2m-client-credentials" + token_exchange = "token-exchange" + id_jag = "id-jag" + aws_sigv4 = "aws-sigv4" + extra_headers = "extra-headers" + not_applicable = "not-applicable" + unresolved = "unresolved" + failed = "resolution-failed" + multiple = "multiple" + + +@dataclass(frozen=True, slots=True) +class ResolvedCredential: + auth: httpx.Auth = field(repr=False) + source: AuthResolution + + class AuthSpecKind(str, Enum): """The server's statically-declared upstream-auth mode — derived from its `config`. diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 9a52da0cab1..b53c55ef5fa 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -49,7 +49,11 @@ from litellm.proxy._experimental.mcp_server.mcp_context import ( _mcp_gateway_server_name, _mcp_proxy_mode, # pyright: ignore[reportPrivateUsage] # server-owned request mode ) -from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug +from litellm.proxy._experimental.mcp_server.mcp_debug import ( + MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, + MCPAuthDiagnostics, + MCPDebug, +) from litellm.proxy._experimental.mcp_server.oauth_utils import ( _redact_mcp_resource_url, get_passthrough_www_authenticate, @@ -4472,13 +4476,13 @@ if MCP_AVAILABLE: raw_headers=raw_headers, scope=dict(scope), mcp_servers=mcp_servers, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, oauth2_headers=oauth2_headers, client_ip=_client_ip, ) - if _debug_headers: - send = MCPDebug.wrap_send_with_debug_headers(send, _debug_headers) + diagnostics: Final = MCPAuthDiagnostics() if _debug_headers else None + if diagnostics is not None: + scope[MCP_AUTH_DIAGNOSTICS_SCOPE_KEY] = diagnostics + send = MCPDebug.wrap_send_with_debug_headers(send, _debug_headers, diagnostics.headers) # Ensure session managers are initialized if not _SESSION_MANAGERS_INITIALIZED: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py index 6b3098d9e60..2eab053ab73 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py @@ -1203,3 +1203,69 @@ async def test_passthrough_ignores_the_carrier_and_keeps_the_callers_slot(): assert isinstance(result, Ok) headers, _ = await _emitted_async(result.ok) assert headers["Authorization"] == "caller-token" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("config", "subject", "expected_source", "expected_header"), + [ + (NoneConfig(), _SUBJECT, "no-auth", None), + (PassthroughConfig(), _SUBJECT, "no-auth", None), + (PassthroughConfig(), _with_inbound("Bearer caller-token"), "oauth2-passthrough", "Bearer caller-token"), + (ApiKeyConfig(key_source=SharedKey(value=SecretStr("static-key"))), _SUBJECT, "static-token", "Bearer static-key"), + (AuthorizationCodeConfig(), Subject(tenant_id="", subject_id="alice"), "stored-user-token", "Bearer stored-alice"), + ], +) +async def test_resolved_source_matches_the_credential_sent_upstream(config, subject, expected_source, expected_header): + from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source + + store = _FakeTokenStore({("alice", "s"): OAuthToken(access_token="stored-alice")}) + provider = UpstreamCredentialProvider(oauth_token_store=store) + result = await resolve_credentials_with_source(provider, subject, _spec(config)) + assert isinstance(result, Ok) + assert result.ok.source.value == expected_source + assert _emitted(result.ok.auth).get("Authorization") == expected_header + assert "stored-alice" not in repr(result.ok) + assert "static-key" not in repr(result.ok) + + +@pytest.mark.asyncio +async def test_resolved_source_preserves_missing_user_token_error(): + from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source + + result = await resolve_credentials_with_source(UpstreamCredentialProvider(), _SUBJECT, _spec(AuthorizationCodeConfig())) + assert isinstance(result, Error) + assert result.error.tag == "unauthorized" + + +@pytest.mark.asyncio +async def test_minted_token_sources_match_egress_and_do_not_fetch_twice(): + from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source + + source = _FakeM2MSource(Ok(OAuthToken(access_token="m2m-at"))) + m2m = await resolve_credentials_with_source(UpstreamCredentialProvider(client_credentials_source=source), _SUBJECT, _spec(_M2M)) + assert isinstance(m2m, Ok) + headers, _ = await _emitted_async(m2m.ok.auth) + assert headers["Authorization"] == "Bearer m2m-at" + assert m2m.ok.source.value == "m2m-client-credentials" + assert source.gets == ["s"] + + exchanger = _FakeExchanger(Ok(OAuthToken(access_token="exchanged-at"))) + exchanged = await resolve_credentials_with_source(UpstreamCredentialProvider(token_exchanger=exchanger), _with_inbound("subject"), _spec(_OBO)) + assert isinstance(exchanged, Ok) + assert _emitted(exchanged.ok.auth)["Authorization"] == "Bearer exchanged-at" + assert exchanged.ok.source.value == "token-exchange" + assert len(exchanger.calls) == 1 + + +@pytest.mark.asyncio +async def test_id_jag_source_describes_final_token_after_both_exchanges(): + from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source + + endpoint = _FakeTokenEndpoint(_two_leg_ok("resource-token")) + provider = UpstreamCredentialProvider(token_endpoint=endpoint) + result = await resolve_credentials_with_source(provider, _with_inbound("identity-token"), _spec(_id_jag_config())) + assert isinstance(result, Ok) + assert result.ok.source.value == "id-jag" + assert _emitted(result.ok.auth)["Authorization"] == "Bearer resource-token" + assert len(endpoint.calls) == 2 diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_debug.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_debug.py index d299239f68e..5039209c351 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_debug.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_debug.py @@ -3,11 +3,17 @@ Tests for MCPDebug — MCP OAuth2 debug response headers. """ import asyncio -from unittest.mock import MagicMock +from typing import Final + +import pytest +from starlette.types import Message + +from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution from litellm.proxy._experimental.mcp_server.mcp_debug import ( MCP_DEBUG_REQUEST_HEADER, MCPDebug, + MCPAuthDiagnostics, ) @@ -166,77 +172,6 @@ class TestBuildDebugHeaders: assert set(headers.keys()) == expected_keys -class TestResolveAuthResolution: - def _make_server(self, **kwargs): - server = MagicMock() - server.alias = kwargs.get("alias", "test") - server.server_name = kwargs.get("server_name", "test") - server.has_client_credentials = kwargs.get("has_client_credentials", False) - server.authentication_token = kwargs.get("authentication_token", None) - server.auth_type = kwargs.get("auth_type", None) - return server - - def test_per_request_header(self): - server = self._make_server() - result = MCPDebug.resolve_auth_resolution( - server, - mcp_auth_header="Bearer xxx", - mcp_server_auth_headers=None, - oauth2_headers=None, - ) - assert result == "per-request-header" - - def test_server_specific_header(self): - server = self._make_server(alias="atlas") - result = MCPDebug.resolve_auth_resolution( - server, - mcp_auth_header=None, - mcp_server_auth_headers={"atlas": {"Authorization": "Bearer xxx"}}, - oauth2_headers=None, - ) - assert result == "per-request-header" - - def test_m2m(self): - server = self._make_server(has_client_credentials=True) - result = MCPDebug.resolve_auth_resolution( - server, - mcp_auth_header=None, - mcp_server_auth_headers=None, - oauth2_headers=None, - ) - assert result == "m2m-client-credentials" - - def test_static_token(self): - server = self._make_server(authentication_token="static-tok") - result = MCPDebug.resolve_auth_resolution( - server, - mcp_auth_header=None, - mcp_server_auth_headers=None, - oauth2_headers=None, - ) - assert result == "static-token" - - def test_oauth2_passthrough(self): - server = self._make_server(auth_type="oauth2") - result = MCPDebug.resolve_auth_resolution( - server, - mcp_auth_header=None, - mcp_server_auth_headers=None, - oauth2_headers={"Authorization": "Bearer eyJ..."}, - ) - assert result == "oauth2-passthrough" - - def test_no_auth(self): - server = self._make_server() - result = MCPDebug.resolve_auth_resolution( - server, - mcp_auth_header=None, - mcp_server_auth_headers=None, - oauth2_headers=None, - ) - assert result == "no-auth" - - class TestWrapSendWithDebugHeaders: def test_injects_headers(self): captured = [] @@ -269,3 +204,85 @@ class TestWrapSendWithDebugHeaders: asyncio.run(wrapped(body_msg)) assert captured[0] == body_msg + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", tuple(AuthResolution)) +async def test_debug_uses_resolution_recorded_after_response_start(source: AuthResolution) -> None: + captured: Final[list[Message]] = [] + diagnostics: Final = MCPAuthDiagnostics() + + async def send(message: Message) -> None: + captured.append(message) + + wrapped: Final = MCPDebug.wrap_send_with_debug_headers(send, {}, diagnostics.headers) + await wrapped({"type": "http.response.start", "status": 200, "headers": []}) + assert captured == [] + diagnostics.record("s1", source) + body: Final[Message] = {"type": "http.response.body", "body": b"data: pong\n\n", "more_body": True} + await wrapped(body) + assert dict(captured[0]["headers"])[b"x-mcp-debug-auth-resolution"] == source.value.encode() + assert captured[1] == body + + +@pytest.mark.asyncio +async def test_early_stream_frame_reports_unresolved_without_waiting() -> None: + captured: Final[list[Message]] = [] + diagnostics: Final = MCPAuthDiagnostics() + + async def send(message: Message) -> None: + captured.append(message) + + wrapped: Final = MCPDebug.wrap_send_with_debug_headers(send, {}, diagnostics.headers) + await wrapped({"type": "http.response.start", "status": 200, "headers": []}) + await wrapped({"type": "http.response.body", "body": b": ping\n\n", "more_body": True}) + diagnostics.record("s1", AuthResolution.stored_user_token) + await wrapped({"type": "http.response.body", "body": b"data: pong\n\n", "more_body": False}) + assert len(captured) == 3 + assert dict(captured[0]["headers"])[b"x-mcp-debug-auth-resolution"] == b"unresolved" + + +def test_diagnostics_keep_requests_separate_and_do_not_collapse_multiple_servers() -> None: + alice: Final = MCPAuthDiagnostics() + bob: Final = MCPAuthDiagnostics() + alice.record("s1", AuthResolution.stored_user_token) + assert bob.resolution() == "unresolved" + alice.record("s1", AuthResolution.token_exchange) + assert alice.resolution() == "token-exchange" + alice.record("s2", AuthResolution.static_token) + assert alice.resolution() == "multiple" + assert alice.headers()["x-mcp-debug-auth-resolutions"] == '{"s1":"token-exchange","s2":"static-token"}' + + +@pytest.mark.asyncio +async def test_concurrent_mcp_messages_record_on_their_own_http_scope() -> None: + from unittest.mock import MagicMock + + from mcp.server.lowlevel.server import request_ctx + from mcp.shared.context import RequestContext + from starlette.requests import Request + + from litellm.proxy._experimental.mcp_server.mcp_debug import ( + MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, + record_auth_resolution, + ) + + session: Final = MagicMock() + first: Final = MCPAuthDiagnostics() + second: Final = MCPAuthDiagnostics() + + async def record(diagnostics: MCPAuthDiagnostics, source: AuthResolution) -> None: + context: Final = RequestContext( + request_id=1, meta=None, session=session, lifespan_context=None, + request=Request({"type": "http", MCP_AUTH_DIAGNOSTICS_SCOPE_KEY: diagnostics}), + ) + token: Final = request_ctx.set(context) + try: + await asyncio.sleep(0) + record_auth_resolution("same-server", source) + finally: + request_ctx.reset(token) + + await asyncio.gather(record(first, AuthResolution.stored_user_token), record(second, AuthResolution.per_request_header)) + assert first.resolution() == "stored-user-token" + assert second.resolution() == "per-request-header" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 46fef83092d..99447acd371 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -12567,3 +12567,109 @@ async def test_pre_call_tool_check_honors_guardrail_attached_to_key(monkeypatch, with pytest.raises(HTTPException) as exc_info: await call assert exc_info.value.status_code == 400 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("config", "extra_headers", "expected_source", "expected_authorization"), + [ + ("stored", None, "stored-user-token", "Bearer stored-token"), + ("stored", {"aUtHoRiZaTiOn": "Bearer injected"}, "stored-user-token", "Bearer stored-token"), + ("static", None, "static-token", "Bearer static-token"), + ("static", {"authorization": "Bearer injected"}, "extra-headers", "Bearer injected"), + ("none", None, "no-auth", None), + ("none", {"Authorization": "Bearer injected"}, "extra-headers", "Bearer injected"), + ], +) +async def test_debug_resolution_matches_final_header_conflict_winner( + config, extra_headers, expected_source, expected_authorization +): + from mcp.server.lowlevel.server import request_ctx + from mcp.shared.context import RequestContext + from starlette.requests import Request + from pydantic import SecretStr + + from litellm.proxy._experimental.mcp_server.auth.litellm_auth_handler import MCPAuthenticatedUser + from litellm.proxy._experimental.mcp_server.mcp_debug import MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, MCPAuthDiagnostics + from litellm.proxy._experimental.mcp_server.outbound_credentials import ( + ApiKeyConfig, AuthorizationCodeConfig, NoneConfig, ServerSpec, SharedKey, UpstreamCredentialProvider, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import OAuthToken + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + class Store: + def __init__(self): + self.calls = 0 + + async def fetch(self, user_id: str, server_id: str) -> OAuthToken | None: + self.calls += 1 + return OAuthToken(access_token="stored-token") if user_id == "alice" else None + + store = Store() + context = MCPAuthenticatedUser(UserAPIKeyAuth(user_id="alice")) + diagnostics = MCPAuthDiagnostics() + token = request_ctx.set(RequestContext( + request_id=1, meta=None, session=MagicMock(), lifespan_context=None, + request=Request({"type": "http", MCP_AUTH_DIAGNOSTICS_SCOPE_KEY: diagnostics}), + )) + selected = { + "stored": AuthorizationCodeConfig(), + "static": ApiKeyConfig(key_source=SharedKey(value=SecretStr("static-token"))), + "none": NoneConfig(), + }[config] + try: + auth, remaining = await MCPServerManager()._resolve_v2_auth( + server=MCPServer( + server_id="s", name="s", transport="http", url="https://up.example/mcp", + static_headers={"Authorization": "Bearer configured"}, + ), + spec=ServerSpec(server_id="s", resource="https://up.example/mcp", config=selected), + provider=UpstreamCredentialProvider(oauth_token_store=store), + subject_token=None, + user_api_key_auth=context.user_api_key_auth, + extra_headers=extra_headers, + ) + request = httpx.Request("GET", "https://up.example/mcp", headers=remaining) + if auth is not None: + next(auth.auth_flow(request)) + assert diagnostics.resolution() == expected_source + assert request.headers.get("Authorization") == expected_authorization + assert store.calls == (1 if config == "stored" else 0) + finally: + request_ctx.reset(token) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", ["http", "stdio"]) +async def test_debug_reports_legacy_signing_and_non_http_transport(transport): + from mcp.server.lowlevel.server import request_ctx + from mcp.shared.context import RequestContext + from starlette.requests import Request + + from litellm.proxy._experimental.mcp_server.mcp_debug import MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, MCPAuthDiagnostics + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + diagnostics = MCPAuthDiagnostics() + token = request_ctx.set(RequestContext( + request_id=1, meta=None, session=MagicMock(), lifespan_context=None, + request=Request({"type": "http", MCP_AUTH_DIAGNOSTICS_SCOPE_KEY: diagnostics}), + )) + try: + server = MCPServer( + server_id="signed", name="signed", transport=transport, + url="https://up.example/mcp", auth_type="aws_sigv4", + aws_access_key_id="AKIDEXAMPLE", aws_secret_access_key="test-signing-secret", + aws_region_name="us-east-1", aws_service_name="execute-api", + command="python", args=["-c", "pass"], + ) + client = await MCPServerManager()._create_mcp_client(server) + if transport == "stdio": + assert diagnostics.resolution() == "not-applicable" + else: + assert diagnostics.resolution() == "aws-sigv4" + request = httpx.Request("POST", "https://up.example/mcp", content=b"{}") + next(client._aws_auth.auth_flow(request)) + assert request.headers["Authorization"].startswith("AWS4-HMAC-SHA256 ") + assert "Credential=AKIDEXAMPLE/" in request.headers["Authorization"] + finally: + request_ctx.reset(token) From c5e93aff132685582679ed1ce7097e7e855630a7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:45:26 -0700 Subject: [PATCH 217/411] fix(proxy): warn at submit when a body-selected post_call policy is deferred --- litellm/proxy/utils.py | 17 ++++++++++ .../proxy_logging/test_guardrail_pipeline.py | 31 +++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1604dea1003..1f8a4d5ba33 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -622,6 +622,15 @@ def _defer_post_call_pipelines( response.id, ", ".join(tag_matched), ) + body_selected: Final = _body_selected_deferrals(data, deferred) + if body_selected: + verbose_proxy_logger.warning( + "Policy engine: background response %s matched post_call policies through the request body's policies " + "list at submit; retrieval carries no request body, so those policies do not govern the completed " + "response: %s", + response.id, + ", ".join(body_selected), + ) _withdraw_deferred_claims(data, deferred) @@ -638,6 +647,14 @@ def _tag_matched_deferrals( ) +def _body_selected_deferrals( + data: Mapping[str, object], deferred: Sequence[tuple[str, "GuardrailPipeline"]] +) -> tuple[str, ...]: + sources: Final = _policy_state_metadata(data).get("policy_sources") + attributed: Final = frozenset(sources) if isinstance(sources, dict) else frozenset() + return tuple(policy_name for policy_name, _pipeline in deferred if policy_name not in attributed) + + def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool: unsupported: Final = tuple( dict.fromkeys( diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 3bbdbbe2ccd..cc40706c67c 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -941,6 +941,7 @@ def _post_call_pipeline_data( "metadata": { "_guardrail_pipelines": [("response-governance", pipeline)], "_pipeline_managed_guardrails": {guardrail}, + "policy_sources": {"response-governance": "model:m"}, }, **extra, } @@ -1468,7 +1469,7 @@ def _output_passing_callbacks() -> list[CustomGuardrail]: def _claimed_post_call_pipeline_data( - *policy_names: str, extra_guardrails: dict[str, list[str]] | None = None, policy_source: str = "model:m" + *policy_names: str, extra_guardrails: dict[str, list[str]] | None = None, policy_source: str | None = "model:m" ): from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -1491,7 +1492,7 @@ def _claimed_post_call_pipeline_data( "_pipeline_managed_guardrails": {"gr-post"}, "applied_policies": list(policy_names), "applied_guardrails": ["gr-post", *(g for gs in (extra_guardrails or {}).values() for g in gs)], - "policy_sources": {policy_name: policy_source for policy_name in policy_names}, + "policy_sources": {policy_name: policy_source for policy_name in policy_names if policy_source is not None}, }, } @@ -1551,6 +1552,32 @@ async def test_pending_background_response_warns_when_the_deferred_policy_was_ma ] +@pytest.mark.asyncio +async def test_pending_background_response_warns_when_the_deferred_policy_came_from_the_request_body( + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + monkeypatch: pytest.MonkeyPatch, + clear_policy_registry: None, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(litellm, "callbacks", _output_blocking_callbacks({})) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False) + data = _claimed_post_call_pipeline_data("body-governance", policy_source=None) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + out = await proxy_logging.post_call_success_hook( + data=data, response=_background_response("queued"), user_api_key_dict=make_user_api_key_auth() + ) + + assert out.status == "queued" + assert "policy_sources" not in data["metadata"] + assert _warnings(caplog) == [ + "Policy engine: background response resp_bg matched post_call policies through the request body's policies " + "list at submit; retrieval carries no request body, so those policies do not govern the completed " + "response: body-governance" + ] + + @pytest.mark.asyncio async def test_pending_background_response_matched_through_its_model_does_not_warn( proxy_logging: ProxyLogging, From 5bdf45726d0a27372c1cc34ee6baa85ef5439650 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:52:08 -0700 Subject: [PATCH 218/411] fix(bedrock): honor the json_mode kwarg on invoke Nova structured output and keep gpt-4o-mini off chat web search --- .../bedrock/chat/converse_transformation.py | 10 +++-- ...odel_prices_and_context_window_backup.json | 6 +-- model_prices_and_context_window.json | 6 +-- .../chat/test_converse_transformation.py | 38 +++++++++++++++++++ whitelisted_bedrock_models.txt | 1 + 5 files changed, 50 insertions(+), 11 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index fa24f8be893..6173f08fc87 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1805,6 +1805,7 @@ class AmazonConverseConfig(BaseConfig): data=request_data, messages=messages, encoding=encoding, + json_mode=json_mode, ) def _transform_reasoning_content(self, reasoning_content_blocks: list[BedrockConverseReasoningContentBlock]) -> str: @@ -2237,6 +2238,7 @@ class AmazonConverseConfig(BaseConfig): data: dict | str, messages: list, encoding, + json_mode: bool | None = None, ) -> ModelResponse: ## LOGGING if logging_obj is not None: @@ -2247,7 +2249,9 @@ class AmazonConverseConfig(BaseConfig): additional_args={"complete_input_dict": data}, ) - json_mode: Final[bool | None] = optional_params.get("json_mode", None) + resolved_json_mode: Final[bool | None] = ( + json_mode if json_mode is not None else optional_params.get("json_mode", None) + ) ## RESPONSE OBJECT try: completion_response: Final = ConverseResponseBlock(**response.json()) @@ -2339,7 +2343,7 @@ class AmazonConverseConfig(BaseConfig): chat_completion_message["thinking_blocks"] = self._transform_thinking_blocks(reasoningContentBlocks) chat_completion_message["content"] = content_str filtered_tools: Final = self._filter_json_mode_tools( - json_mode=json_mode, + json_mode=resolved_json_mode, tools=tools, chat_completion_message=chat_completion_message, ) @@ -2363,7 +2367,7 @@ class AmazonConverseConfig(BaseConfig): # When json_mode filtered out all synthetic tool calls the response # is plain content, not a pending tool invocation. Fix finish_reason # so callers (e.g. OpenAI SDK) don't misinterpret it. - if json_mode and not filtered_tools and tools: + if resolved_json_mode and not filtered_tools and tools: initial_finish_reason = "stop" ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 885021692c9..a2e7b692649 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29407,8 +29407,7 @@ "search_context_size_high": 0.025, "search_context_size_low": 0.025, "search_context_size_medium": 0.025 - }, - "supports_web_search": true + } }, "gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, @@ -29436,8 +29435,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true + "supports_vision": true }, "gpt-4o-mini-audio-preview": { "deprecation_date": "2026-05-07", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 885021692c9..a2e7b692649 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29407,8 +29407,7 @@ "search_context_size_high": 0.025, "search_context_size_low": 0.025, "search_context_size_medium": 0.025 - }, - "supports_web_search": true + } }, "gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, @@ -29436,8 +29435,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true + "supports_vision": true }, "gpt-4o-mini-audio-preview": { "deprecation_date": "2026-05-07", diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index f0e361ceb88..16824d19f23 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -6727,3 +6727,41 @@ def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( ) assert result == {"any": {}} + + +def test_transform_response_honors_json_mode_kwarg_when_optional_params_lack_it(): + response_json = { + "metrics": {"latencyMs": 900}, + "output": { + "message": { + "content": [ + { + "toolUse": { + "input": {"city": "Paris", "population": 2100000}, + "name": "json_tool_call", + "toolUseId": "tooluse_invoke_nova_json", + } + } + ], + "role": "assistant", + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 40, "outputTokens": 20, "totalTokens": 60}, + } + raw_response = httpx.Response(200, json=response_json, request=httpx.Request("POST", "https://bedrock.test")) + logging_obj = MagicMock() + result = AmazonConverseConfig().transform_response( + model="bedrock/invoke/us.amazon.nova-micro-v1:0", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=logging_obj, + request_data={}, + messages=[], + optional_params={"tools": [{"type": "function", "function": {"name": "json_tool_call", "parameters": {}}}]}, + litellm_params={}, + encoding=None, + json_mode=True, + ) + assert result.choices[0].message.tool_calls is None + assert json.loads(result.choices[0].message.content) == {"city": "Paris", "population": 2100000} diff --git a/whitelisted_bedrock_models.txt b/whitelisted_bedrock_models.txt index 578edddec8d..6124cb41044 100644 --- a/whitelisted_bedrock_models.txt +++ b/whitelisted_bedrock_models.txt @@ -6,6 +6,7 @@ ai21.jamba-instruct-v1:0 twelvelabs.pegasus-1-2-v1:0 us.twelvelabs.pegasus-1-2-v1:0 eu.twelvelabs.pegasus-1-2-v1:0 +global.twelvelabs.pegasus-1-2-v1:0 amazon.titan-text-express-v1 amazon.titan-text-lite-v1 amazon.titan-text-premier-v1:0 From b89c32407f81141e6c877b534862ad5b3e7956e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:01:36 -0700 Subject: [PATCH 219/411] test(cost): assert gpt-4o-mini keeps chat web search off while the flat search fee still matches its alias --- .../llm_cost_calc/test_tool_call_cost_tracking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index f319413045f..bbb7b5f9c35 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -879,8 +879,8 @@ def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias( alias_info = litellm.get_model_info("gpt-4o-mini") snapshot_info = litellm.get_model_info("gpt-4o-mini-2024-07-18") - assert snapshot_info["supports_web_search"] is True - assert alias_info["supports_web_search"] is True + assert not snapshot_info["supports_web_search"] + assert not alias_info["supports_web_search"] snapshot_cost = StandardBuiltInToolCostTracking.get_cost_for_web_search( web_search_options=web_search_options, model_info=snapshot_info From 29af8b734913f5e43e4149bcdcc36a7c9261cc37 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:04:54 -0700 Subject: [PATCH 220/411] fix(streaming_handler): replay a cached completion with no choices as an empty stream A stream cache hit on an entry stored with choices == [] indexed choices[0] in the cached_response branch and failed with IndexError, so the streaming converters' empty chunk had no working consumer. The branch now treats a chunk without choices as empty and lets the wrapper close the stream with its usual finish_reason stop chunk --- .../litellm_core_utils/streaming_handler.py | 11 ++--- .../test_streaming_handler.py | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6ae17bac6ff..db23929e0c3 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1473,17 +1473,14 @@ class CustomStreamWrapper: self.received_finish_reason = response_obj["finish_reason"] elif self.custom_llm_provider == "cached_response": cached_chunk: Final = cast(ModelResponseStream, chunk) - chunk_finish_reason: Final = cached_chunk.choices[0].finish_reason + cached_choice: Final = cached_chunk.choices[0] if cached_chunk.choices else None + chunk_finish_reason: Final = cached_choice.finish_reason if cached_choice is not None else None response_obj = { - "text": cached_chunk.choices[0].delta.content, + "text": cached_choice.delta.content if cached_choice is not None else None, "is_finished": chunk_finish_reason is not None, "finish_reason": chunk_finish_reason, "original_chunk": cached_chunk, - "tool_calls": ( - cached_chunk.choices[0].delta.tool_calls - if hasattr(cached_chunk.choices[0].delta, "tool_calls") - else None - ), + "tool_calls": (getattr(cached_choice.delta, "tool_calls", None) if cached_choice is not None else None), } completion_obj["content"] = response_obj["text"] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 0aa73833677..ced7d6d677d 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -6,7 +6,7 @@ import pytest import asyncio import traceback -from typing import Optional +from typing import Final, Optional import litellm from litellm import verbose_logger @@ -2633,6 +2633,47 @@ def test_dispatch_cached_response_extracts_delta( assert initialized_custom_stream_wrapper.response_id == "chatcmpl-cache-1" +def test_dispatch_cached_response_without_choices_is_an_empty_chunk( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """A cached completion with no choices replays as an empty, unfinished chunk + instead of raising IndexError on choices[0].""" + initialized_custom_stream_wrapper.custom_llm_provider = "cached_response" + chunk: Final = ModelResponseStream(id="chatcmpl-cache-empty", choices=[]) + + result, model_response, completion_obj = _run_dispatch( + initialized_custom_stream_wrapper, chunk + ) + + assert isinstance(result, _ProviderChunkParsed) + assert completion_obj["content"] is None + assert initialized_custom_stream_wrapper.received_finish_reason is None + assert model_response.id == "chatcmpl-cache-empty" + + +@pytest.mark.asyncio +async def test_cached_response_without_choices_streams_a_single_stop_chunk( + logging_obj: Logging, +): + """A stream cache hit on a completion stored with choices == [] ends with one + finish_reason=stop chunk, the same shape the live empty stream produced.""" + + async def cached_chunks(): + yield ModelResponseStream(id="chatcmpl-cache-empty", choices=[]) + + wrapper: Final = CustomStreamWrapper( + completion_stream=cached_chunks(), + model="test-model", + logging_obj=logging_obj, + custom_llm_provider="cached_response", + ) + + chunks: Final = tuple([chunk async for chunk in wrapper]) + + assert tuple(choice.finish_reason for chunk in chunks for choice in chunk.choices) == ("stop",) + assert all(choice.delta.content in (None, "") for chunk in chunks for choice in chunk.choices) + + def test_dispatch_vertex_ai_legacy_text_and_finish_reason( initialized_custom_stream_wrapper: CustomStreamWrapper, ): From f49ebc93eaf108d2312779346485ce6808c26184 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:11:43 -0700 Subject: [PATCH 221/411] fix(anthropic): request encrypted reasoning only where the Responses provider returns it The bridge now asks for reasoning.encrypted_content whenever the provider's Responses config lists include, independent of the client's thinking block, and leaves it out for providers such as Perplexity that reject the param. Bridge-tagged blocks are stripped on the chat adapter path too, so a mid session model switch to Gemini or Bedrock no longer forwards them as real signatures, a bare prefix counts as bridge-tagged, and non-mapping messages pass through the strip untouched. --- .../prompt_templates/common_utils.py | 18 ++++++--- litellm/llms/anthropic/common_utils.py | 2 + .../adapters/transformation.py | 4 +- .../responses_adapters/handler.py | 17 ++++++++- .../responses_adapters/transformation.py | 9 ++++- ...ore_utils_prompt_templates_common_utils.py | 3 ++ ...al_pass_through_adapters_transformation.py | 38 +++++++++++++++++++ .../test_responses_adapters_handler.py | 23 +++++++++++ .../test_responses_adapters_transformation.py | 12 +++++- .../anthropic/test_anthropic_common_utils.py | 9 +++++ 10 files changed, 126 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 918cb982773..75a3aefa98b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1853,22 +1853,30 @@ def encrypted_reasoning_signature(encrypted_content: str) -> str: return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}" +def _carries_encrypted_reasoning(signature: object) -> bool: + return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX) + + def encrypted_content_from_signature(signature: object) -> str | None: - if not isinstance(signature, str) or not signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX): + if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature): return None return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None -def _encrypted_content_of_block(block: Mapping[str, object]) -> str | None: +def _encrypted_reasoning_field(block: Mapping[str, object]) -> object: match block.get("type"): case "thinking": - return encrypted_content_from_signature(block.get("signature")) + return block.get("signature") case "redacted_thinking": - return encrypted_content_from_signature(block.get("data")) + return block.get("data") case _: return None +def _encrypted_content_of_block(block: Mapping[str, object]) -> str | None: + return encrypted_content_from_signature(_encrypted_reasoning_field(block)) + + def is_encrypted_reasoning_block(block: object) -> bool: """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. @@ -1878,7 +1886,7 @@ def is_encrypted_reasoning_block(block: object) -> bool: if not isinstance(block, Mapping): return False mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance - return _encrypted_content_of_block(mapping) is not None + return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping)) def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index cb24f4153f9..ca520116606 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1203,6 +1203,8 @@ def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[A def _without_encrypted_reasoning_blocks(message: dict) -> dict | None: # mutable-ok: Anthropic message payload shape + if not isinstance(message, Mapping): + return message content: Final = message.get("content") if not isinstance(content, list): return message diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index db890662132..aaffc84b872 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -113,6 +113,7 @@ from litellm.litellm_core_utils.reasoning_effort_utils import ( from litellm.llms.anthropic.common_utils import ( is_empty_unsigned_thinking_block, normalize_anthropic_tool_use_id, + strip_encrypted_reasoning_blocks_from_anthropic_messages, ) from litellm.llms.anthropic.experimental_pass_through.context_management import ( PolyfillResult, @@ -417,7 +418,8 @@ class LiteLLMAnthropicMessagesAdapter: model: str | None = None, ) -> list: new_messages: Final[list[AllMessageValues]] = [] - for m in messages: + replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + for m in replayable_messages: user_message: ChatCompletionUserMessage | None = None tool_message_list: list[ChatCompletionToolMessage] = [] new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = [] diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 0445c23ed8c..24825cc87e0 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.utils import ProviderConfigManager from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper @@ -34,6 +35,15 @@ def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, return extra_kwargs or {} +def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool: + provider: Final = ( + custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1] + ) + provider_model: Final = local_model_name(model, provider) + responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(provider, provider_model) + return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) + + def _build_responses_kwargs( *, max_tokens: int, @@ -85,8 +95,13 @@ def _build_responses_kwargs( request_data["output_format"] = output_format anthropic_request: Final = AnthropicMessagesRequest(**request_data) - responses_kwargs: Final = _ADAPTER.translate_request(anthropic_request) forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + responses_kwargs: Final = _ADAPTER.translate_request( + anthropic_request, + include_encrypted_reasoning=_provider_returns_encrypted_reasoning( + model, forwarded_kwargs.get("custom_llm_provider") + ), + ) # Normalize reasoning effort based on model capabilities # (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 07419db443c..1fdb0318bab 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -505,10 +505,16 @@ class LiteLLMAnthropicToResponsesAPIAdapter: def translate_request( self, anthropic_request: AnthropicMessagesRequest, + include_encrypted_reasoning: bool = True, ) -> dict[str, Any]: """ Translate a full Anthropic /v1/messages request dict to litellm.responses() / litellm.aresponses() kwargs. + + ``include_encrypted_reasoning`` asks the provider for ``reasoning.encrypted_content`` + on every call, so a reasoning model's items can be replayed intact next turn even + when the client sent no ``thinking`` block; pass False for a provider whose + Responses API rejects ``include``. """ model: Final[str] = anthropic_request["model"] messages_list: Final = cast( @@ -538,6 +544,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: "model": model, "input": input_items, } + if include_encrypted_reasoning: + responses_kwargs["include"] = [RESPONSES_INCLUDE_ENCRYPTED_REASONING] # mutable-ok: API request payload if system and not developer_parts: if isinstance(system, str): @@ -582,7 +590,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) if reasoning: responses_kwargs["reasoning"] = reasoning - responses_kwargs["include"] = [RESPONSES_INCLUDE_ENCRYPTED_REASONING] # mutable-ok: json list # output_format / output_config.format -> text format # output_format: {"type": "json_schema", "schema": {...}} diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 9df124576a0..ecb07d8c937 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -7,6 +7,7 @@ import pytest from litellm.litellm_core_utils.prompt_templates.common_utils import ( + ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, add_system_prompt_to_messages, @@ -1610,6 +1611,8 @@ class TestEncryptedReasoningReplay: [ ({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True), ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True), + ({"type": "thinking", "thinking": "x", "signature": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), + ({"type": "redacted_thinking", "data": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), ({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False), ({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False), ({"type": "text", "text": encrypted_reasoning_signature("g")}, False), diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 30465ca25ba..279ed7a8b33 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -9,6 +9,7 @@ import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, + encrypted_reasoning_signature, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, @@ -407,6 +408,43 @@ def test_translate_anthropic_messages_to_openai_thinking_blocks(): assert result[1]["tool_calls"][0]["id"] == "toolu_01234" +def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning_blocks(): + """A session that moves from an OpenAI reasoning model to a chat provider replays reasoning only OpenAI can read. + + Gemini rejects the whole request when such a block reaches it as a thought_signature, so the + adapter drops those blocks and keeps the provider-signed ones. + """ + + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[{"type": "text", "text": "Who drinks water?"}], + ), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + {"type": "text", "text": "The Norwegian."}, + ], + ), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + {"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_signed"}, + {"type": "text", "text": "Still the Norwegian."}, + ], + ), + ] + + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(messages=anthropic_messages) + + assert [m["role"] for m in result] == ["user", "assistant", "assistant"] + assert not result[1].get("thinking_blocks") + assert result[1]["content"] == "The Norwegian." + assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"] + + def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): """Reasoning-aware chat providers read reasoning_content, so thinking text must land there. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 3383813245a..2cbfaa17d23 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -54,6 +54,29 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived() assert responses_kwargs["prompt_cache_key"] == "explicit-key" +def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thinking(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + extra_kwargs={"custom_llm_provider": "openai"}, + ) + assert responses_kwargs["include"] == ["reasoning.encrypted_content"] + assert "reasoning" not in responses_kwargs + + +def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_rejects_it(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="perplexity/sonar", + thinking={"type": "enabled", "budget_tokens": 4096}, + extra_kwargs={"custom_llm_provider": "perplexity"}, + ) + assert "include" not in responses_kwargs + assert "reasoning" in responses_kwargs + + def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key(): responses_kwargs = _build_responses_kwargs( max_tokens=1024, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 79eaf6b60f2..9b7b15ace61 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -1162,7 +1162,6 @@ class TestTranslateRequestBroaderCoverage: req = _make_request(thinking={"type": "disabled"}) kwargs = _ADAPTER.translate_request(req) assert "reasoning" not in kwargs - assert "include" not in kwargs def test_thinking_asks_for_the_encrypted_reasoning(self): """The documented way to get reasoning that survives store=false is to ask for it.""" @@ -1170,6 +1169,17 @@ class TestTranslateRequestBroaderCoverage: kwargs = _ADAPTER.translate_request(req) assert kwargs["include"] == ["reasoning.encrypted_content"] + def test_encrypted_reasoning_is_asked_for_without_a_thinking_block(self): + """A reasoning model reasons whether or not the client sent `thinking`, so the replay needs it either way.""" + kwargs = _ADAPTER.translate_request(_make_request()) + assert kwargs["include"] == ["reasoning.encrypted_content"] + + def test_encrypted_reasoning_is_not_asked_for_when_the_provider_rejects_include(self): + req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) + kwargs = _ADAPTER.translate_request(req, include_encrypted_reasoning=False) + assert kwargs["reasoning"] == {"effort": "high"} + assert "include" not in kwargs + def test_metadata_user_id_mapped_to_user(self): req = _make_request(metadata={"user_id": "user-42"}) kwargs = _ADAPTER.translate_request(req) diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 97d22274b1c..91652c89092 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1600,6 +1600,15 @@ class TestAnthropicThinkingSignatureSelfHeal: assert len(msgs[1]["content"]) == 2 assert len(msgs[2]["content"]) == 4 + def test_strip_encrypted_reasoning_leaves_malformed_messages_for_the_provider_to_reject(self): + """A bare string in messages must reach Anthropic as a 400, not die in the stripper as a 500.""" + from litellm.llms.anthropic.common_utils import ( + strip_encrypted_reasoning_blocks_from_anthropic_messages, + ) + + msgs = ["hi", {"role": "user", "content": "hello"}] + assert strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs) == msgs + def test_strip_empty_text_blocks_treats_null_text_as_empty(self): from litellm.llms.anthropic.common_utils import ( strip_empty_content_blocks_from_anthropic_messages, From 2c836f473c8ef6574ae7a04ffa8434f7715e7e2f Mon Sep 17 00:00:00 2001 From: "cursor[bot]" <206951365+cursor[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:14:24 -0700 Subject: [PATCH 222/411] test(ui): derive reasoning-effort assertion from the anthropic preset (#40456) Preset #40341 pointed the Anthropic family REASONING tier at claude-fable-5-1, but this test still hardcoded claude-opus-5, so the payload it saw no longer matched. Rebase the assertion on ANTHROPIC_PRESET.complexity_router_config.tier_model_configs so a preset refresh flows through instead of redding the suite on staging. Co-authored-by: Cursor Agent Co-authored-by: Krrish Dholakia --- .../src/components/add_model/add_auto_router_tab.test.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 014854ac712..f6e619c5e84 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -1046,9 +1046,7 @@ describe("AddAutoRouterTab", () => { await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ complexity_router_config: { - tier_model_configs: { - REASONING: [{ model_name: "claude-opus-5", litellm_params: { reasoning_effort: "high" } }], - }, + tier_model_configs: ANTHROPIC_PRESET.complexity_router_config.tier_model_configs, }, }); }); From 354365eecaf6c70aba2a649fa324bf0369438f85 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:15:01 -0700 Subject: [PATCH 223/411] test(proxy): give the pipeline-managed native hook audit test its chat completions route --- tests/test_litellm/proxy/test_proxy_logging_hook_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index 42888ca7d54..28ff4571b44 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -688,7 +688,7 @@ async def test_deferred_stream_guardrails_skip_pipeline_managed_native_hook(monk "messages": [{"role": "user", "content": "hi"}], "metadata": {"_guardrail_pipelines": [("response-governance", pipeline)]}, }, - captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234"), + captured_user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/chat/completions"), captured_logging_obj=_streaming_logging_obj(), assembled_response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="hello"))]), cache_hit=False, From babf18f37f383b99e697344019482ff97dfea1c7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:20:56 -0700 Subject: [PATCH 224/411] test(token_counter): type the project ITPM lag test's request body --- tests/test_litellm/proxy/hooks/test_tpm_concurrent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py index 398abf60b54..bdaca9ffc2d 100644 --- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -3682,7 +3682,7 @@ async def test_the_project_itpm_reservation_counts_the_request_off_the_event_loo handler, _cache = rate_limiter stash = get_or_create_request_stash() warm_tokenizer("claude-fable-5") - data: Dict[str, Any] = { + data: dict[str, object] = { "model": "claude-fable-5", "messages": [{"role": "user", "content": text * 100}], } From 0a053d2c8146e4a4739f92d48f1e2bdec75202e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:21:40 -0700 Subject: [PATCH 225/411] test(guardrails): cover streamed tool-call name rewrites on chat and Messages Skipping the name write-back in either handler left every test green; a guardrail that renames a tool call now has a regression test on both the chat chunk path and the Anthropic SSE path --- .../test_anthropic_guardrail_handler.py | 23 +++++++++++++++++++ .../test_openai_guardrail_handler.py | 23 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 64c243343d5..e091355b69d 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -367,6 +367,29 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: assert '"stop_reason": "tool_use"' in raw assert "persim" not in raw + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_tool_use_name_back_into_sse_chunks(self): + class RenameTool(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + for tool_call in inputs.get("tool_calls", []): + tool_call.function.name = "lookup_fruit_reviewed" + return inputs + + handler = AnthropicMessagesHandler() + chunks = self._ended_tool_use_sse_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=RenameTool(guardrail_name="test"), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + raw = b"".join(chunks).decode() + assert '"name": "lookup_fruit_reviewed"' in raw and '"id": "toolu_1"' in raw + assert '"name": "lookup_fruit"' not in raw + assert json.loads("".join(self._partial_jsons(chunks))) == {"fruit": "persimmon"} + @pytest.mark.asyncio async def test_ended_stream_tool_use_rewrite_leaves_chunks_untouched_by_default(self): handler = AnthropicMessagesHandler() diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index c7011a6f00e..5a29a96829f 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1171,6 +1171,29 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert chunks[3].choices[0].delta.tool_calls is None assert chunks[3].choices[0].finish_reason == "tool_calls" + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_tool_call_name_back_into_chunks(self): + class RenameTool(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + for tool_call in inputs.get("tool_calls", []): + tool_call["function"]["name"] = "lookup_fruit_reviewed" + return inputs + + handler = OpenAIChatCompletionsHandler() + chunks = self._ended_tool_call_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=RenameTool(guardrail_name="test"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + fragments = [chunk.choices[0].delta.tool_calls[0] for chunk in chunks[:3]] + assert [fragment.function.name for fragment in fragments] == ["lookup_fruit_reviewed", None, None] + assert json.loads("".join(fragment.function.arguments for fragment in fragments)) == {"fruit": "persimmon"} + assert fragments[0].id == "call_1" + @pytest.mark.asyncio async def test_ended_stream_tool_call_rewrite_leaves_chunks_untouched_by_default(self): handler = OpenAIChatCompletionsHandler() From cce1d2087b9ae997ec406d6d38857642f1461844 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:22:22 -0700 Subject: [PATCH 226/411] fix(responses): echo a named tool_choice in the Responses API shape on the chat-completions bridge A streamed /v1/responses request with tool_choice {"type": "function", "name": ...} that reaches a chat-completions-only deployment failed with HTTP 500 before the first byte: the synthetic response.created and response.in_progress events copied the chat-shaped tool_choice into ResponsesAPIResponse, whose ToolChoice type expects the flat Responses API shape. The non-streamed path echoed "auto" regardless of the request. Both paths now normalize the request's tool_choice through the existing chat transform and map it back to the Responses API vocabulary, validated by a TypeAdapter(ToolChoice), so a named function is echoed as {"type": "function", "name": ...} and a missing tool_choice is echoed as "auto". Fixes #33689 --- .../streaming_iterator.py | 11 ++-- .../transformation.py | 18 ++++++- .../test_litellm_completion_responses.py | 52 ++++++++++++++++++ .../test_streaming_iterator_transformation.py | 53 +++++++++++++++++++ 4 files changed, 126 insertions(+), 8 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index d7f8cd8f8bd..660dd8f0c92 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -437,14 +437,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): response_created_event_data["temperature"] = self.responses_api_request["temperature"] if "text" in self.responses_api_request: response_created_event_data["text"] = self.responses_api_request["text"] - if "tool_choice" in self.responses_api_request: - # Transform tool_choice from dict format (e.g., {"type": "auto"}) to string format - response_created_event_data["tool_choice"] = ( - LiteLLMCompletionResponsesConfig._transform_tool_choice(self.responses_api_request["tool_choice"]) - or "auto" + response_created_event_data["tool_choice"] = ( + LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( + self.responses_api_request.get("tool_choice") ) - else: - response_created_event_data["tool_choice"] = "auto" + ) if "tools" in self.responses_api_request: response_created_event_data["tools"] = self.responses_api_request["tools"] else: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b2d1a69e0d8..878f493b58e 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -27,6 +27,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import ( ) from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.response_create_params import ResponseInputParam +from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict @@ -68,6 +69,7 @@ from litellm.types.llms.openai import ( ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponsesAPIStatus, + ToolChoice, ValidChatCompletionMessageContentTypes, ValidChatCompletionMessageContentTypesLiteral, ) @@ -126,6 +128,7 @@ _STR_KEY_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _DICT_ITEMS_LIST_ADAPTER: Final = TypeAdapter(list[dict[object, object]]) _TEXT_ADAPTER: Final = TypeAdapter(str) +_RESPONSES_API_TOOL_CHOICE_ADAPTER: Final = TypeAdapter(ToolChoice) @runtime_checkable @@ -267,6 +270,17 @@ class LiteLLMCompletionResponsesConfig: # Return as-is for unknown formats return tool_choice + @staticmethod + def _transform_tool_choice_for_responses_api_response(tool_choice: object) -> ToolChoice: + normalized: Final = LiteLLMCompletionResponsesConfig._transform_tool_choice(tool_choice) + match normalized: + case None: + return "auto" + case {"type": "function", "function": {"name": str(function_name)}}: + return ToolChoiceFunctionParam(type="function", name=function_name) + case _: + return _RESPONSES_API_TOOL_CHOICE_ADAPTER.validate_python(normalized) + @staticmethod def _should_drop_derived_web_search_options(model: str, custom_llm_provider: str | None) -> bool: """ @@ -2263,7 +2277,9 @@ class LiteLLMCompletionResponsesConfig: ), parallel_tool_calls=getattr(chat_completion_response, "parallel_tool_calls", False), temperature=getattr(chat_completion_response, "temperature", 0), - tool_choice=getattr(chat_completion_response, "tool_choice", "auto"), + tool_choice=LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( + responses_api_request.get("tool_choice") + ), tools=getattr(chat_completion_response, "tools", []), top_p=getattr(chat_completion_response, "top_p", None), max_output_tokens=getattr(chat_completion_response, "max_output_tokens", None), diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2068f10ea2d..2aab660b5b7 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1421,6 +1421,58 @@ class TestToolChoiceTransformation: ) assert result == "required" + @pytest.mark.parametrize( + "request_tool_choice,expected", + [ + ({"type": "function", "name": "run_command"}, {"type": "function", "name": "run_command"}), + ({"type": "function", "function": {"name": "run_command"}}, {"type": "function", "name": "run_command"}), + ({"type": "custom", "name": "ApplyPatch"}, {"type": "function", "name": "ApplyPatch"}), + ({"type": "tool"}, "required"), + ({"type": "auto"}, "auto"), + ("required", "required"), + ("none", "none"), + (None, "auto"), + ], + ) + def test_transform_tool_choice_for_responses_api_response(self, request_tool_choice, expected): + result = LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( + request_tool_choice + ) + assert result == expected + + def test_non_streamed_response_echoes_named_tool_choice_in_responses_api_shape(self): + chat_completion_response = ModelResponse( + id="chatcmpl-named-tool-choice", + created=1748575031, + model="claude-haiku-4-5", + object="chat.completion", + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_pwd", + type="function", + function=Function(name="run_command", arguments='{"command":"pwd"}'), + ) + ], + ), + ) + ], + ) + + responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Run the command pwd.", + responses_api_request={"tool_choice": {"type": "function", "name": "run_command"}}, + chat_completion_response=chat_completion_response, + ) + + assert responses_api_response.tool_choice == {"type": "function", "name": "run_command"} + class TestContentTypeTransformation: """Test content type transformation from Responses API to Chat Completion format""" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 719d51c11e3..d4b565f82a1 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -628,3 +628,56 @@ def test_streamed_anthropic_tool_call_events_correlate_on_normalized_item_id(): assert item_dones[0].item.call_id == "toolu_01AbCdEf" for evt in deltas + dones: assert evt.item_id == added[0].item.id + + +def _tool_call_chunk(finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-haiku-4-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_pwd", + "type": "function", + "function": {"name": "run_command", "arguments": '{"command":"pwd"}'}, + "index": 0, + } + ], + ), + finish_reason=finish_reason, + ) + ], + ) + + +def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape(): + iterator = LiteLLMCompletionStreamingIterator( + model="claude-haiku-4-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper([_tool_call_chunk(finish_reason="tool_calls")]), + request_input="Run the command pwd.", + responses_api_request={ + "tools": [{"type": "function", "name": "run_command", "parameters": {"type": "object"}}], + "tool_choice": {"type": "function", "name": "run_command"}, + }, + custom_llm_provider="anthropic", + litellm_metadata={}, + ) + + events = list(iterator) + + response_events = [event for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] + assert [event.type for event in response_events] == [ + "response.created", + "response.in_progress", + "response.completed", + ] + assert response_events[0].response.tool_choice == {"type": "function", "name": "run_command"} + assert response_events[1].response.tool_choice == {"type": "function", "name": "run_command"} + assert any(getattr(event, "type", None) == "response.output_item.done" for event in events) From 5e7cb898065158626fec3a4afb90cb23a411095c Mon Sep 17 00:00:00 2001 From: Praveena Date: Tue, 4 Aug 2026 16:33:28 +0530 Subject: [PATCH 227/411] fix(azure): respect DEFAULT_MAX_RETRIES in initialize_azure_sdk_client When litellm_params does not include max_retries (the common case for deployments configured without explicit retry settings), initialize_azure_sdk_client() previously passed None through the 'if max_retries is not None' guard, resulting in AsyncAzureOpenAI being created without a max_retries argument. The OpenAI SDK then uses its own hardcoded default of 2, ignoring the DEFAULT_MAX_RETRIES env var. Fix: fall back to litellm.constants.DEFAULT_MAX_RETRIES when litellm_params has no max_retries. This ensures the SDK client respects the configured retry count. Steps to reproduce: 1. Set env var DEFAULT_MAX_RETRIES=0 2. Configure a deployment without explicit max_retries in litellm_params 3. Make a request that triggers a timeout 4. Observe: SDK retries (x-stainless-retry-count=1) despite env var=0 Related: https://github.com/BerriAI/litellm/issues/5124 --- litellm/llms/azure/common_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 6cb7d09cec4..f20cc37572b 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -582,7 +582,11 @@ class BaseAzureLLM(BaseOpenAILLM): if scope is None: scope = "https://cognitiveservices.azure.com/.default" - max_retries: Final = litellm_params.get("max_retries") + max_retries = litellm_params.get("max_retries") + if max_retries is None: + from litellm.constants import DEFAULT_MAX_RETRIES + + max_retries = DEFAULT_MAX_RETRIES timeout: Final = litellm_params.get("timeout") if not api_key and azure_ad_token_provider is None and tenant_id and client_id and client_secret: verbose_logger.debug("Using Azure AD Token Provider from Entra ID for Azure Auth") From 830ee2d39d84eba4d2c8a27851e363473af038d0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:29:06 -0700 Subject: [PATCH 228/411] fix(azure): default max_retries to DEFAULT_MAX_RETRIES with regression tests initialize_azure_sdk_client now falls back to litellm.constants.DEFAULT_MAX_RETRIES when litellm_params carries no max_retries, so off-router Azure clients (files, batches, fine-tuning, assistants, audio) honor the env var like OpenAI clients do. Router paths already default max_retries to 0 and are unchanged. Regression tests cover the default, explicit 0/5/None values, and the env var reaching the SDK client in a fresh interpreter. --- litellm/llms/azure/common_utils.py | 11 ++-- .../llms/azure/test_azure_common_utils.py | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index f20cc37572b..e6b3eb1f2bb 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -14,6 +14,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.caching.caching import DualCache +from litellm.constants import DEFAULT_MAX_RETRIES from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.common_utils import BaseOpenAILLM from litellm.secret_managers.get_azure_ad_token_provider import ( @@ -582,11 +583,8 @@ class BaseAzureLLM(BaseOpenAILLM): if scope is None: scope = "https://cognitiveservices.azure.com/.default" - max_retries = litellm_params.get("max_retries") - if max_retries is None: - from litellm.constants import DEFAULT_MAX_RETRIES - - max_retries = DEFAULT_MAX_RETRIES + configured_max_retries: Final = litellm_params.get("max_retries") + max_retries: Final = DEFAULT_MAX_RETRIES if configured_max_retries is None else configured_max_retries timeout: Final = litellm_params.get("timeout") if not api_key and azure_ad_token_provider is None and tenant_id and client_id and client_secret: verbose_logger.debug("Using Azure AD Token Provider from Entra ID for Azure Auth") @@ -646,8 +644,7 @@ class BaseAzureLLM(BaseOpenAILLM): else: azure_client_params["http_client"] = self._get_sync_http_client() - if max_retries is not None: - azure_client_params["max_retries"] = max_retries + azure_client_params["max_retries"] = max_retries if timeout is not None: azure_client_params["timeout"] = timeout diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index f000abb4c9a..c959c201ccb 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -385,6 +385,58 @@ def test_select_azure_base_url_called(setup_mocks): setup_mocks["select_url"].assert_called_once() +def test_initialize_defaults_max_retries_to_litellm_default(setup_mocks): + result = BaseAzureLLM().initialize_azure_sdk_client( + litellm_params={}, + api_key="test-api-key", + api_base="https://test.openai.azure.com", + model_name="gpt-4", + api_version="2023-06-01", + is_async=False, + ) + + assert result["max_retries"] == litellm.constants.DEFAULT_MAX_RETRIES + + +@pytest.mark.parametrize( + "configured, expected", + [(0, 0), (5, 5), (None, litellm.constants.DEFAULT_MAX_RETRIES)], +) +def test_initialize_honors_explicit_max_retries(setup_mocks, configured, expected): + result = BaseAzureLLM().initialize_azure_sdk_client( + litellm_params={"max_retries": configured}, + api_key="test-api-key", + api_base="https://test.openai.azure.com", + model_name="gpt-4", + api_version="2023-06-01", + is_async=False, + ) + + assert result["max_retries"] == expected + + +def test_default_max_retries_env_var_reaches_azure_sdk_client(): + import subprocess + import sys + + code = ( + "from litellm.llms.azure.common_utils import BaseAzureLLM\n" + "client = BaseAzureLLM().get_azure_openai_client(" + "api_key='test-api-key', api_base='https://test.openai.azure.com', api_version='2024-02-01'," + " client=None, _is_async=True, litellm_params={}, model='gpt-4')\n" + "print(client.max_retries)" + ) + completed = subprocess.run( + [sys.executable, "-c", code], + env={**os.environ, "DEFAULT_MAX_RETRIES": "0"}, + capture_output=True, + text=True, + check=True, + ) + + assert completed.stdout.strip() == "0" + + @pytest.mark.parametrize( "call_type", [ From 3020a13e24f56a4068055eaadce039cd527f7faa Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:34:12 -0700 Subject: [PATCH 229/411] test(e2e): every spend row a virtual key writes joins its token across all write paths One aliased key owned by a user with an email drives chat, queued chat, messages, responses, embeddings, the Gemini passthrough, a batch file upload, and a batch create against a live proxy. Each row must carry api_key equal to the key's LiteLLM_VerificationToken.token and the alias in metadata, and /spend/logs?api_key= and /user/daily/activity must report the key with its alias and email. Health-check rows must keep the literal service-account key, and the batch cost row for a completed marker batch must join the key that created it. A re-hashed api_key (the v1.99.0 regression fixed by #39568 and #39572) now fails the Buildkite e2e stage naming the write path Resolves MAT-180 --- tests/e2e/CLAUDE.md | 5 +- .../coverage_registry/quota_management.yaml | 4 + tests/e2e/models.py | 2 + .../spend_tracking/conftest.py | 1 + .../spend_tracking/spend_e2e_client.py | 284 +++++++++++++- .../test_key_attribution_e2e.py | 349 ++++++++++++++++++ 6 files changed, 642 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 89c04208d65..b92d84b8b25 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -181,13 +181,14 @@ quota_management... | team_multi_window | fallback | spend_counter chat_completions | stream | messages_bridge | embeddings | cache_hit | key_rollup | concurrent_burst | tags | end_user - | per_model | failure | spend_calculate | pagination + | per_model | failure | spend_calculate | pagination | key_attribution assertion : blocks_over_limit | resets_after_window | headers_report_remaining | picks_under_tpm | blocks_then_resets | resets_windows_independently | alerts_without_blocking | isolates_per_model | isolates_per_member | isolates_per_group | enforced_across_keys | routes_to_fallback | reseed_matches_db | reports_spend | logs_cost | zero_cost | matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows - | writes_failure_row | returns_cost | keeps_total + | writes_failure_row | returns_cost | keeps_total | joins_key | reports_alias_and_email + | health_rows_keep_service_account | batch_cost_joins_key e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages] quota_management.budget.key.blocks_over_limit exercised_on=[chat_completions] ``` diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index d0afcaca848..7013b59b83d 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -58,3 +58,7 @@ - {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} - {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} - {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"} +- {id: quota_management.spend_tracking.key_attribution.joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [joins_key], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Every spend row a virtual key writes across chat, queued chat, messages, responses, embeddings, the Gemini passthrough, file upload, and batch create carries api_key equal to the key's token hash and the key alias, the join the usage APIs depend on; a re-hashed token shows up as an unattributed key-hash-* row (#39568, #39572)"} +- {id: quota_management.spend_tracking.key_attribution.reports_alias_and_email, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [reports_alias_and_email], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native], source: "proxy/management_endpoints/internal_user_endpoints.py", rationale: "/spend/logs?api_key= returns every one of the key's rows with its alias and /user/daily/activity aggregates them under the key's token with key_alias and user_email; /spend/logs carries no email field, so the email is asserted on daily activity only"} +- {id: quota_management.spend_tracking.key_attribution.health_rows_keep_service_account, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [health_rows_keep_service_account], exercised_on: [chat_completions], source: "proxy/health_check.py", rationale: "A /health probe's spend row stays keyed by the literal litellm-internal-health-check service account rather than a hash of it, so health spend never appears as an unattributed key"} +- {id: quota_management.spend_tracking.key_attribution.batch_cost_joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [batch_cost_joins_key], exercised_on: [batches], source: "enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py", rationale: "The _batch_cost row the batch cost poller writes once a batch completes carries the submitting key's token hash; it rides a cross-run marker batch whose metadata records the expected hash, since completion can lag by up to the polling interval. Cold start with no completed marker is a documented vacuous pass"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 62810e6cfd9..faf8557498b 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -84,6 +84,7 @@ class KeyGenerateBody(BaseModel): class KeyGenerateResponse(BaseModel): key: str + token: str | None = None key_alias: str | None = None models: list[str] = [] max_budget: float | None = None @@ -672,6 +673,7 @@ class GuardrailRunRecord(BaseModel): class SpendLogMetadata(BaseModel): + user_api_key_alias: str | None = None applied_guardrails: list[str] | None = None guardrail_information: list[GuardrailRunRecord] | None = None diff --git a/tests/e2e/quota_management/spend_tracking/conftest.py b/tests/e2e/quota_management/spend_tracking/conftest.py index 0597c9af400..9c8ffd18144 100644 --- a/tests/e2e/quota_management/spend_tracking/conftest.py +++ b/tests/e2e/quota_management/spend_tracking/conftest.py @@ -36,6 +36,7 @@ DRIVER_MODELS: tuple[tuple[str, str, str], ...] = ( ("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY"), ("openai-text-embedding-3-small", "openai/text-embedding-3-small", "OPENAI_API_KEY"), ("openai-responses-codex", "openai/gpt-5.3-codex", "OPENAI_API_KEY"), + ("openai-gpt-4o-mini", "openai/gpt-4o-mini", "OPENAI_API_KEY"), ) diff --git a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py index 056799b8499..42ebfdb98af 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py +++ b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py @@ -15,9 +15,14 @@ import time from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from typing import Final + +from pydantic import BaseModel, Field from e2e_config import unique_marker from e2e_http import ( + FileUploadForm, + Headers, NoBody, ProbeResult, Result, @@ -35,6 +40,8 @@ from models import ( DateRangeParams, EmbedBody, EmbedResponse, + KeyGenerateBody, + KeyGenerateResponse, OpenAPISchema, SpendCalculateBody, SpendCalculateResponse, @@ -43,13 +50,24 @@ from models import ( SpendLogsPageParams, SpendTagsResponse, TagSpend, + UserDeleteBody, + UserDeleteResponse, + UserNewBody, + UserNewResponse, + UserRole, ) -from proxy_client import ProxyClient +from proxy_client import Converged, ProxyClient, await_converged __all__ = [ + "BatchCreateBody", + "BatchObject", + "DailyActivityKeyBreakdown", + "FileObject", "ProbeResult", + "ResponseIdentity", "SpendClient", "SpendLogRow", + "StreamingResponse", "build_client", "is_ok", "unique_marker", @@ -57,6 +75,112 @@ __all__ = [ ] +class GeminiApiKeyHeaders(Headers): + x_goog_api_key: str = Field(serialization_alias="x-goog-api-key") + content_type: str = Field(default="application/json", serialization_alias="Content-Type") + + +class GeminiPart(BaseModel): + text: str + + +class GeminiContent(BaseModel): + parts: list[GeminiPart] + + +class GeminiGenerationConfig(BaseModel): + maxOutputTokens: int + + +class GeminiGenerateBody(BaseModel): + contents: list[GeminiContent] + generationConfig: GeminiGenerationConfig + + +class ResponsesBody(BaseModel): + model: str + input: str + cache: dict[str, bool] | None = {"no-cache": True} + + +class QueuedChatBody(ChatBody): + priority: int = 0 + + +class ResponseIdentity(BaseModel): + id: str | None = None + + +class HealthParams(BaseModel): + model: str + + +class ModelQuery(BaseModel): + model: str + + +class FileObject(BaseModel): + id: str + + +class BatchCreateBody(BaseModel): + input_file_id: str + endpoint: str = "/v1/chat/completions" + completion_window: str = "24h" + model: str + metadata: dict[str, str] + + +class BatchObject(BaseModel): + id: str + status: str + metadata: dict[str, str] | None = None + created_at: int | None = None + + +class BatchList(BaseModel): + data: list[BatchObject] = [] + + +class BatchListQuery(BaseModel): + model: str + limit: int + + +class DailyActivityParams(BaseModel): + start_date: str + end_date: str + api_key: str + + +class DailyActivityKeyMetadata(BaseModel): + key_alias: str | None = None + team_id: str | None = None + user_email: str | None = None + + +class DailyActivityKeyMetrics(BaseModel): + api_requests: int = 0 + + +class DailyActivityKeyBreakdown(BaseModel): + metrics: DailyActivityKeyMetrics + metadata: DailyActivityKeyMetadata + + +class DailyActivityBreakdown(BaseModel): + api_keys: dict[str, DailyActivityKeyBreakdown] = {} + + +class DailyActivityRow(BaseModel): + date: str + breakdown: DailyActivityBreakdown + + +class DailyActivityResponse(BaseModel): + results: list[DailyActivityRow] = [] + + def _chat_body( model: str, content: str, @@ -207,6 +331,164 @@ class SpendClient: def probe(self, path: str, *, params: DateRangeParams) -> ProbeResult: return self.proxy.transport.probe(path, params=params) + def create_user(self, *, email: str, role: UserRole, user_id: str) -> str: + return unwrap( + self.proxy.transport.post( + "/user/new", + headers=self.proxy.transport.master, + json=UserNewBody(user_email=email, user_role=role, user_id=user_id), + response_type=UserNewResponse, + ) + ).user_id + + def delete_user(self, user_id: str) -> None: + _ = self.proxy.transport.post( + "/user/delete", + headers=self.proxy.transport.master, + json=UserDeleteBody(user_ids=[user_id]), + response_type=UserDeleteResponse, + ) + + def generate_key_record(self, body: KeyGenerateBody) -> KeyGenerateResponse: + return unwrap( + self.proxy.transport.post( + "/key/generate", + headers=self.proxy.transport.master, + json=body, + response_type=KeyGenerateResponse, + ) + ) + + def send_chat(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: + return self.proxy.transport.send( + "/chat/completions", + headers=self.proxy.transport.bearer(key), + json=_chat_body(model, content, max_tokens=max_tokens), + ) + + def send_queued_chat(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: + return self.proxy.transport.send( + "/queue/chat/completions", + headers=self.proxy.transport.bearer(key), + json=QueuedChatBody( + model=model, + messages=[ChatMessage(role="user", content=content)], + max_tokens=max_tokens, + ), + ) + + def send_messages(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: + return self.proxy.transport.send( + "/v1/messages", + headers=self.proxy.transport.bearer(key), + json=AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=content)], + max_tokens=max_tokens, + ), + ) + + def send_responses(self, key: str, model: str, content: str) -> StreamingResponse: + return self.proxy.transport.send( + "/v1/responses", + headers=self.proxy.transport.bearer(key), + json=ResponsesBody(model=model, input=content), + ) + + def send_embed(self, key: str, model: str, content: str) -> StreamingResponse: + return self.proxy.transport.send( + "/embeddings", + headers=self.proxy.transport.bearer(key), + json=EmbedBody(model=model, input=content), + ) + + def send_gemini_generate(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: + return self.proxy.transport.send( + f"/gemini/v1beta/models/{model}:generateContent", + headers=GeminiApiKeyHeaders(x_goog_api_key=key), + json=GeminiGenerateBody( + contents=[GeminiContent(parts=[GeminiPart(text=content)])], + generationConfig=GeminiGenerationConfig(maxOutputTokens=max_tokens), + ), + ) + + def upload_batch_file(self, key: str, model: str, content: bytes) -> FileObject: + return unwrap( + self.proxy.transport.upload( + "/v1/files", + headers=self.proxy.transport.bearer(key), + form=FileUploadForm(purpose="batch"), + filename="key_attribution.jsonl", + content=content, + params=ModelQuery(model=model), + response_type=FileObject, + ) + ) + + def create_batch(self, key: str, body: BatchCreateBody) -> BatchObject: + return unwrap( + self.proxy.transport.post( + "/v1/batches", + headers=self.proxy.transport.bearer(key), + json=body, + response_type=BatchObject, + ) + ) + + def list_batches(self, key: str, model: str, *, limit: int) -> list[BatchObject]: + return unwrap( + self.proxy.transport.get( + "/v1/batches", + headers=self.proxy.transport.bearer(key), + params=BatchListQuery(model=model, limit=limit), + response_type=BatchList, + ) + ).data + + def retrieve_batch(self, key: str, batch_id: str) -> BatchObject: + return unwrap( + self.proxy.transport.get( + f"/v1/batches/{batch_id}", + headers=self.proxy.transport.bearer(key), + params=NoBody(), + response_type=BatchObject, + ) + ) + + def health(self, model: str) -> ProbeResult: + return self.proxy.transport.probe("/health", params=HealthParams(model=model)) + + def daily_activity_for_key(self, token: str, *, start: datetime, end: datetime) -> DailyActivityKeyBreakdown | None: + response: Final = unwrap( + self.proxy.transport.get( + "/user/daily/activity", + headers=self.proxy.transport.master, + params=DailyActivityParams( + start_date=start.strftime("%Y-%m-%d"), + end_date=end.strftime("%Y-%m-%d"), + api_key=token, + ), + response_type=DailyActivityResponse, + ) + ) + return next( + (row.breakdown.api_keys[token] for row in response.results if token in row.breakdown.api_keys), + None, + ) + + def poll_daily_activity_for_key( + self, token: str, *, start: datetime, end: datetime, min_requests: int + ) -> DailyActivityKeyBreakdown | None: + outcome: Final = await_converged( + lambda: self.daily_activity_for_key(token, start=start, end=end), + converged=lambda found: found is not None and found.metrics.api_requests >= min_requests, + timeout=self.proxy.poll_timeout, + interval=self.proxy.poll_interval, + now=time.monotonic, + sleep=time.sleep, + ) + return outcome.result if isinstance(outcome, Converged) else outcome.last_result + def openapi(self) -> OpenAPISchema: return unwrap( self.proxy.transport.get( diff --git a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py new file mode 100644 index 00000000000..986033dbbde --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py @@ -0,0 +1,349 @@ +"""Every spend row a live proxy writes joins its virtual key (MAT-180). + +One virtual key with an alias, owned by a user with an email, drives every spend +write path a key can reach: /chat/completions, /queue/chat/completions, +/v1/messages, /v1/responses, /embeddings, the Gemini native passthrough, a batch +input file upload, and a batch create. Each row those calls write must carry +`api_key` equal to the key's LiteLLM_VerificationToken.token (the sha256 hash +/key/generate returns as `token`), which is the join /spend/logs?api_key= and +/user/daily/activity rely on to report key_alias and user_email. A row keyed by a +re-hashed token (v1.99.0's regression, #39568 and #39572) shows up as a +key-hash-* row with no alias and no email in the customer's usage exports. + +The health-check service account writes rows too; those must stay keyed by the +literal service-account name, never by a hash of it. The batch cost row is +written by the CheckBatchCost poller once the batch completes, up to an hour +later, so it rides a cross-run baton like the batches suite: each run submits a +one-line marker batch whose metadata records the token it expects on the cost +row, and asserts on the newest completed marker from any run (a cold start with +no completed marker is a documented vacuous pass, never a skip). + +/spend/logs carries no email field, so the email assertion lives on +/user/daily/activity alone; /spend/logs is held to the alias in metadata. +""" + +import time +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Final, Iterator + +import pytest + +from models import ChatMessage, KeyGenerateBody +from proxy_client import Converged, await_converged +from spend_e2e_client import ( + BatchCreateBody, + BatchObject, + DailyActivityKeyBreakdown, + ResponseIdentity, + SpendClient, + SpendLogRow, + StreamingResponse, + unique_marker, +) +from pydantic import BaseModel + +pytestmark = pytest.mark.e2e + +CHAT_MODEL: Final = "gemini-2.5-flash" +MESSAGES_MODEL: Final = "claude-haiku-4-5" +RESPONSES_MODEL: Final = "openai-responses-codex" +EMBED_MODEL: Final = "openai-text-embedding-3-small" +BATCH_MODEL: Final = "openai-gpt-4o-mini" +BATCH_BACKEND_MODEL: Final = "gpt-4o-mini" +HEALTH_SERVICE_ACCOUNT: Final = "litellm-internal-health-check" +BATON_MARKER_KEY: Final = "litellm_e2e_suite" +BATON_MARKER_VALUE: Final = "key-attribution-baton" +BATON_EXPECTED_TOKEN_KEY: Final = "expected_api_key" +BATON_POLL_SECONDS: Final = 300.0 +BATON_POLL_INTERVAL_SECONDS: Final = 10.0 +BATON_LIST_LIMIT: Final = 100 +MAX_TOKENS: Final = 8 +WRITE_PATHS: Final = ( + "chat_completions", + "queue_chat_completions", + "messages", + "responses", + "embeddings", + "gemini_passthrough", + "batch_file_upload", + "batch_create", +) + + +class BatchLineBody(BaseModel): + model: str + messages: list[ChatMessage] + max_tokens: int + + +class BatchLine(BaseModel): + custom_id: str + method: str = "POST" + url: str = "/v1/chat/completions" + body: BatchLineBody + + +@dataclass(frozen=True, slots=True) +class AttributedKey: + key: str + token: str + alias: str + email: str + user_id: str + + +@dataclass(frozen=True, slots=True) +class WritePath: + name: str + request_id: str + + +@dataclass(frozen=True, slots=True) +class DrivenKey: + identity: AttributedKey + paths: tuple[WritePath, ...] + started_at: datetime + + +def _body_id(name: str, sent: StreamingResponse) -> WritePath: + assert sent.ok, f"{name} failed with {sent.status_code}: {sent.body[:300]}" + response_id: Final = ResponseIdentity.model_validate_json(sent.body).id + assert response_id, f"{name} answered without a response id: {sent.body[:300]}" + return WritePath(name=name, request_id=response_id) + + +def _call_id(name: str, sent: StreamingResponse) -> WritePath: + assert sent.ok, f"{name} failed with {sent.status_code}: {sent.body[:300]}" + assert sent.call_id, f"{name} answered without an x-litellm-call-id header" + return WritePath(name=name, request_id=sent.call_id) + + +def _batch_jsonl(marker: str) -> bytes: + line: Final = BatchLine( + custom_id=marker, + body=BatchLineBody( + model=BATCH_BACKEND_MODEL, + messages=[ChatMessage(role="user", content=f"Reply with the word ok. {marker}")], + max_tokens=MAX_TOKENS, + ), + ) + return f"{line.model_dump_json()}\n".encode() + + +def _drive_batch(client: SpendClient, identity: AttributedKey, marker: str) -> tuple[WritePath, WritePath]: + uploaded: Final = client.upload_batch_file(identity.key, BATCH_MODEL, _batch_jsonl(marker)) + created: Final = client.create_batch( + identity.key, + BatchCreateBody( + input_file_id=uploaded.id, + model=BATCH_MODEL, + metadata={ + BATON_MARKER_KEY: BATON_MARKER_VALUE, + BATON_EXPECTED_TOKEN_KEY: identity.token, + "run": marker, + }, + ), + ) + return ( + WritePath(name="batch_file_upload", request_id=uploaded.id), + WritePath(name="batch_create", request_id=created.id), + ) + + +def _drive_every_write_path(client: SpendClient, identity: AttributedKey) -> tuple[WritePath, ...]: + marker: Final = unique_marker() + prompt: Final = f"Reply with the word ok. {marker}" + key: Final = identity.key + return ( + _body_id("chat_completions", client.send_chat(key, CHAT_MODEL, prompt, max_tokens=MAX_TOKENS)), + _body_id("queue_chat_completions", client.send_queued_chat(key, CHAT_MODEL, prompt, max_tokens=MAX_TOKENS)), + _body_id("messages", client.send_messages(key, MESSAGES_MODEL, prompt, max_tokens=MAX_TOKENS)), + _body_id("responses", client.send_responses(key, RESPONSES_MODEL, prompt)), + _call_id("embeddings", client.send_embed(key, EMBED_MODEL, prompt)), + _call_id("gemini_passthrough", client.send_gemini_generate(key, CHAT_MODEL, prompt, max_tokens=MAX_TOKENS)), + *_drive_batch(client, identity, marker), + ) + + +def _completed_baton(listed: list[BatchObject]) -> BatchObject | None: + return max( + ( + batch + for batch in listed + if batch.status == "completed" and (batch.metadata or {}).get(BATON_MARKER_KEY) == BATON_MARKER_VALUE + ), + key=lambda batch: batch.created_at or 0, + default=None, + ) + + +def _newest_completed_baton(client: SpendClient, key: str) -> BatchObject | None: + outcome: Final = await_converged( + lambda: _completed_baton(client.list_batches(key, BATCH_MODEL, limit=BATON_LIST_LIMIT)), + converged=lambda completed: completed is not None, + timeout=BATON_POLL_SECONDS, + interval=BATON_POLL_INTERVAL_SECONDS, + now=time.monotonic, + sleep=time.sleep, + ) + return outcome.result if isinstance(outcome, Converged) else None + + +def _health_rows_between(client: SpendClient, started_at: datetime) -> list[SpendLogRow]: + return [ + row + for row in client.proxy.spend_logs_window( + start=started_at - timedelta(minutes=1), end=datetime.now(timezone.utc) + timedelta(minutes=1) + ) + if HEALTH_SERVICE_ACCOUNT in (row.request_tags or []) + ] + + +def _health_rows_since(client: SpendClient, started_at: datetime) -> list[SpendLogRow]: + outcome: Final = await_converged( + lambda: _health_rows_between(client, started_at), + converged=lambda rows: bool(rows), + timeout=client.proxy.poll_timeout, + interval=client.proxy.poll_interval, + now=time.monotonic, + sleep=time.sleep, + ) + return outcome.result if isinstance(outcome, Converged) else outcome.last_result + + +class TestKeyAttribution: + @pytest.fixture(scope="class") + def driven(self, client: SpendClient) -> Iterator[DrivenKey]: + marker: Final = unique_marker() + user_id: Final = client.create_user( + email=f"key-attribution-{marker}@example.com", + role="proxy_admin", + user_id=f"key-attribution-{marker}", + ) + record: Final = client.generate_key_record( + KeyGenerateBody(models=[], user_id=user_id, key_alias=f"key-attribution-{marker}") + ) + assert record.token, "/key/generate answered without the key's token hash" + assert record.key_alias, "/key/generate dropped the key alias" + identity: Final = AttributedKey( + key=record.key, + token=record.token, + alias=record.key_alias, + email=f"key-attribution-{marker}@example.com", + user_id=user_id, + ) + started_at: Final = datetime.now(timezone.utc) + try: + yield DrivenKey( + identity=identity, + paths=_drive_every_write_path(client, identity), + started_at=started_at, + ) + finally: + client.proxy.delete_key(identity.key) + client.delete_user(identity.user_id) + + @pytest.mark.covers( + "quota_management.spend_tracking.key_attribution.joins_key", + exercised_on=["chat_completions", "messages", "responses", "embeddings", "batches", "files", "google_native"], + ) + def test_every_write_path_row_joins_the_key(self, client: SpendClient, driven: DrivenKey) -> None: + assert tuple(path.name for path in driven.paths) == WRITE_PATHS + found: Final = tuple( + (path, client.proxy.poll_logs_for_request_id(path.request_id)) for path in driven.paths + ) + unwritten: Final = [path.name for path, rows in found if not rows] + assert not unwritten, f"write paths that produced no spend row within the poll window: {unwritten}" + unjoined: Final = [ + (path.name, row.call_type, row.api_key) + for path, rows in found + for row in rows + if row.api_key != driven.identity.token + ] + assert not unjoined, ( + "spend rows whose api_key does not join LiteLLM_VerificationToken.token " + f"{driven.identity.token}: {unjoined}" + ) + unaliased: Final = [ + (path.name, row.call_type, row.metadata.user_api_key_alias if row.metadata else None) + for path, rows in found + for row in rows + if row.metadata is None or row.metadata.user_api_key_alias != driven.identity.alias + ] + assert not unaliased, f"spend rows written without key alias {driven.identity.alias!r}: {unaliased}" + + @pytest.mark.covers( + "quota_management.spend_tracking.key_attribution.reports_alias_and_email", + exercised_on=["chat_completions", "messages", "responses", "embeddings", "batches", "files", "google_native"], + ) + def test_spend_logs_by_key_return_every_row_with_the_alias(self, client: SpendClient, driven: DrivenKey) -> None: + expected_ids: Final = frozenset(path.request_id for path in driven.paths) + rows: Final = client.poll_logs_for_key( + driven.identity.key, + min_rows=len(driven.paths), + predicate=lambda found: expected_ids <= frozenset(row.request_id or "" for row in found), + ) + missing: Final = expected_ids - frozenset(row.request_id or "" for row in rows) + assert not missing, ( + f"/spend/logs?api_key= does not return {len(missing)} of {len(expected_ids)} rows for the key: " + f"{sorted(path.name for path in driven.paths if path.request_id in missing)}" + ) + aliases: Final = frozenset(row.metadata.user_api_key_alias if row.metadata else None for row in rows) + assert aliases == {driven.identity.alias}, f"/spend/logs rows carry aliases {sorted(map(str, aliases))}" + + @pytest.mark.covers( + "quota_management.spend_tracking.key_attribution.reports_alias_and_email", + exercised_on=["chat_completions", "messages", "responses", "embeddings", "batches", "files", "google_native"], + ) + def test_user_daily_activity_reports_alias_and_email(self, client: SpendClient, driven: DrivenKey) -> None: + breakdown: Final[DailyActivityKeyBreakdown | None] = client.poll_daily_activity_for_key( + driven.identity.token, + start=driven.started_at - timedelta(days=1), + end=datetime.now(timezone.utc) + timedelta(days=1), + min_requests=len(driven.paths), + ) + assert breakdown is not None, ( + f"/user/daily/activity?api_key={driven.identity.token} has no api_keys breakdown: " + "the key's rows did not aggregate under its token" + ) + assert breakdown.metrics.api_requests >= len(driven.paths), ( + f"/user/daily/activity counts {breakdown.metrics.api_requests} requests for the key, " + f"expected at least {len(driven.paths)}" + ) + assert breakdown.metadata.key_alias == driven.identity.alias, f"key_alias={breakdown.metadata.key_alias!r}" + assert breakdown.metadata.user_email == driven.identity.email, f"user_email={breakdown.metadata.user_email!r}" + + @pytest.mark.covers( + "quota_management.spend_tracking.key_attribution.health_rows_keep_service_account", + exercised_on=["chat_completions"], + ) + def test_health_check_rows_keep_the_service_account_key(self, client: SpendClient) -> None: + started_at: Final = datetime.now(timezone.utc) + probe: Final = client.health(CHAT_MODEL) + assert probe.healthy, f"/health?model={CHAT_MODEL} answered {probe.status_code}: {probe.body[:300]}" + rows: Final = _health_rows_since(client, started_at) + assert rows, f"/health?model={CHAT_MODEL} wrote no {HEALTH_SERVICE_ACCOUNT}-tagged spend row" + rehashed: Final = [(row.request_id, row.api_key) for row in rows if row.api_key != HEALTH_SERVICE_ACCOUNT] + assert not rehashed, f"health-check rows keyed by something other than {HEALTH_SERVICE_ACCOUNT!r}: {rehashed}" + + @pytest.mark.covers( + "quota_management.spend_tracking.key_attribution.batch_cost_joins_key", + exercised_on=["batches"], + ) + def test_completed_batch_cost_row_joins_the_key(self, client: SpendClient, driven: DrivenKey) -> None: + completed: Final = _newest_completed_baton(client, driven.identity.key) + if completed is None: + return + expected_token: Final = (completed.metadata or {}).get(BATON_EXPECTED_TOKEN_KEY) + assert expected_token, f"marker batch {completed.id} lost its expected token metadata: {completed.metadata}" + fetched: Final = client.retrieve_batch(driven.identity.key, completed.id) + assert fetched.status == "completed", f"listed-completed marker retrieved as {fetched.status!r}" + rows: Final = client.proxy.poll_logs_for_request_id( + f"{fetched.id}_batch_cost", + predicate=lambda found: any((row.spend or 0) > 0 for row in found), + ) + priced: Final = [row for row in rows if (row.spend or 0) > 0] + assert priced, f"completed batch {fetched.id} has no positive-cost spend row under {fetched.id}_batch_cost" + unjoined: Final = [(row.call_type, row.api_key) for row in priced if row.api_key != expected_token] + assert not unjoined, f"batch cost rows whose api_key does not join the key's token {expected_token}: {unjoined}" From 2488f84b0293956b3736fcd9cf639491ab68e65e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:53:35 -0700 Subject: [PATCH 230/411] fix(proxy): keep a body litellm_session_id in SpendLogs under missing_session_id omit (#40379) * fix(proxy): keep a body litellm_session_id in SpendLogs under missing_session_id omit Under general_settings.missing_session_id: omit, apply_missing_session_id_policy now mirrors a client-supplied top-level litellm_session_id into metadata.session_id when the client did not set one there, so SpendLogs.session_id and Langfuse agree with the session callbacks already report through StandardLoggingPayload.session_id Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep client metadata.session_id ahead of body litellm_session_id on litellm_metadata routes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): drop docstrings from the missing_session_id omit regression tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/litellm_pre_call_utils.py | 10 ++ .../proxy/test_litellm_pre_call_utils.py | 101 ++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 3250ae5cca9..2925226f235 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -773,6 +773,16 @@ def apply_missing_session_id_policy( return if policy == "omit": metadata[SESSION_ID_OMITTED_METADATA_KEY] = True + requester_metadata: Final = data.get("metadata") + requester_session_id: Final = ( + requester_metadata.get("session_id") if isinstance(requester_metadata, dict) else None + ) + if ( + (body_session_id := data.get("litellm_session_id")) + and not metadata.get("session_id") + and not requester_session_id + ): + metadata["session_id"] = body_session_id return if data.get("litellm_session_id") or metadata.get("session_id"): return diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 892fa484ab4..7564fa91c43 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -7678,6 +7678,107 @@ async def test_missing_session_id_omit_keeps_client_supplied_session_id(): assert _spend_log_session_id(updated) == "client-session-1" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "client_body", + [ + {"model": "gpt-4o", "messages": [], "litellm_session_id": "cust-sess-1"}, + {"model": "gpt-4o", "messages": [], "litellm_session_id": "cust-sess-1", "metadata": {"trace_id": "trace-1"}}, + ], +) +async def test_missing_session_id_omit_keeps_body_litellm_session_id( + monkeypatch: pytest.MonkeyPatch, client_body: dict[str, object] +): + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + + updated = await add_litellm_data_to_request( + data=client_body, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + callback_session_id = StandardLoggingPayloadSetup.get_standard_logging_payload_session_id( + logging_obj=SimpleNamespace(litellm_session_id=""), + litellm_params=get_litellm_params(litellm_session_id="cust-sess-1", metadata=updated["metadata"]), + ) + assert callback_session_id == "cust-sess-1" + assert updated["metadata"]["session_id"] == "cust-sess-1" + assert _spend_log_session_id(updated) == "cust-sess-1" + + +@pytest.mark.asyncio +async def test_missing_session_id_omit_body_litellm_session_id_does_not_override_metadata_session_id(): + updated = await add_litellm_data_to_request( + data={ + "model": "gpt-4o", + "messages": [], + "litellm_session_id": "cust-sess-1", + "metadata": {"session_id": "meta-sess-1"}, + }, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert updated["metadata"]["session_id"] == "meta-sess-1" + assert _spend_log_session_id(updated) == "meta-sess-1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/messages"]) +async def test_missing_session_id_omit_keeps_metadata_session_id_on_litellm_metadata_routes(path: str): + updated = await add_litellm_data_to_request( + data={ + "model": "gpt-4o", + "input": "hi", + "litellm_session_id": "cust-sess-1", + "metadata": {"session_id": "meta-sess-1"}, + }, + request=_request_for(path), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert updated["litellm_metadata"]["session_id"] == "meta-sess-1" + assert _spend_log_session_id(updated, "litellm_metadata") == "meta-sess-1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/messages"]) +async def test_missing_session_id_omit_keeps_body_litellm_session_id_on_litellm_metadata_routes(path: str): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "input": "hi", "litellm_session_id": "cust-sess-1"}, + request=_request_for(path), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert updated["litellm_metadata"]["session_id"] == "cust-sess-1" + assert _spend_log_session_id(updated, "litellm_metadata") == "cust-sess-1" + + +@pytest.mark.asyncio +async def test_missing_session_id_omit_ignores_empty_body_litellm_session_id(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [], "litellm_session_id": ""}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "omit"}, + ) + + assert "session_id" not in updated["metadata"] + assert _spend_log_session_id(updated) is None + + @pytest.mark.asyncio async def test_missing_session_id_generate_reuses_traceparent_trace_id(): """A W3C traceparent already decides SpendLogs.session_id, so the callback session id must reuse it.""" From d909c101ab884b13399548f3749bbab26cbed251 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 9 Sep 2026 15:08:21 -0700 Subject: [PATCH 231/411] fix(bedrock): forward aws_session_tags in batch cancel cancel_batch resolved credentials and polled job status without the deployment's aws_session_tags, so on a tag-gated trust policy batch create and retrieve succeeded while cancel failed with AccessDenied. Thread the tags through both calls and cover it with a regression test that fakes STS behind a tag-gated trust policy. --- litellm/llms/bedrock/batches/handler.py | 6 +- .../llms/bedrock/batches/test_handler.py | 84 ++++++++++++++----- 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/litellm/llms/bedrock/batches/handler.py b/litellm/llms/bedrock/batches/handler.py index f73d5f5d3cb..b408d2f620c 100644 --- a/litellm/llms/bedrock/batches/handler.py +++ b/litellm/llms/bedrock/batches/handler.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, cast @@ -6,6 +6,7 @@ from openai.types.batch import BatchRequestCounts from openai.types.batch import Metadata as OpenAIBatchMetadata from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix +from litellm.types.llms.bedrock import AwsSessionTag from litellm.types.utils import LiteLLMBatch if TYPE_CHECKING: @@ -116,6 +117,7 @@ class BedrockBatchesHandler: aws_web_identity_token: str | None = None, aws_sts_endpoint: str | None = None, aws_external_id: str | None = None, + aws_session_tags: Sequence[AwsSessionTag] | None = None, **kwargs: object, # kwargs-ok: litellm.cancel_batch forwards arbitrary user kwargs verbatim ) -> "LiteLLMBatch": try: @@ -139,6 +141,7 @@ class BedrockBatchesHandler: aws_web_identity_token=aws_web_identity_token, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=aws_session_tags, ) client: Final = boto3.client( @@ -163,6 +166,7 @@ class BedrockBatchesHandler: aws_web_identity_token=aws_web_identity_token, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, + aws_session_tags=aws_session_tags, ) try: diff --git a/tests/test_litellm/llms/bedrock/batches/test_handler.py b/tests/test_litellm/llms/bedrock/batches/test_handler.py index 4ccddc33e77..03daafcad72 100644 --- a/tests/test_litellm/llms/bedrock/batches/test_handler.py +++ b/tests/test_litellm/llms/bedrock/batches/test_handler.py @@ -482,10 +482,36 @@ def test_litellm_cancel_batch_dispatches_to_bedrock(patched_boto3): assert batch.status == "cancelled" +class _TagGatedSTSClient: + """Stands in for STS behind a trust policy that only admits sessions carrying ``tags``.""" + + def __init__(self, tags: list[dict[str, str]], access_key_id: str) -> None: + self._tags = tags + self._access_key_id = access_key_id + + def get_caller_identity(self): + return {"Arn": "arn:aws:iam::111111111111:user/litellm-proxy-pod"} + + def assume_role(self, **params): + from botocore.exceptions import ClientError + + if list(params.get("Tags", ())) != self._tags: + raise ClientError( + {"Error": {"Code": "AccessDenied", "Message": "is not authorized to perform: sts:TagSession"}}, + "AssumeRole", + ) + return { + "Credentials": { + "AccessKeyId": self._access_key_id, + "SecretAccessKey": "assumed-secret", + "SessionToken": "assumed-session-token", + "Expiration": datetime.now(timezone.utc) + timedelta(minutes=30), + } + } + + def test_handle_model_invocation_job_status_builds_the_client_from_the_tagged_session(monkeypatch): """Status polling must assume the role with the deployment's session tags, like every other call.""" - from botocore.exceptions import ClientError - monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False) monkeypatch.delenv("AWS_ROLE_ARN", raising=False) tags = [{"Key": "team", "Value": "genai"}] @@ -493,28 +519,9 @@ def test_handle_model_invocation_job_status_builds_the_client_from_the_tagged_se fake_bedrock = MagicMock() fake_bedrock.get_model_invocation_job.return_value = _fake_boto3_response() - class FakeSTSClient: - def get_caller_identity(self): - return {"Arn": "arn:aws:iam::111111111111:user/litellm-proxy-pod"} - - def assume_role(self, **params): - if list(params.get("Tags", ())) != tags: - raise ClientError( - {"Error": {"Code": "AccessDenied", "Message": "is not authorized to perform: sts:TagSession"}}, - "AssumeRole", - ) - return { - "Credentials": { - "AccessKeyId": "ASIABATCHSTATUSTAGGED", - "SecretAccessKey": "assumed-secret", - "SessionToken": "assumed-session-token", - "Expiration": datetime.now(timezone.utc) + timedelta(minutes=30), - } - } - def boto3_client(service_name, **kwargs): if service_name == "sts": - return FakeSTSClient() + return _TagGatedSTSClient(tags, "ASIABATCHSTATUSTAGGED") bedrock_client_kwargs.append(kwargs) return fake_bedrock @@ -530,3 +537,36 @@ def test_handle_model_invocation_job_status_builds_the_client_from_the_tagged_se assert batch.status == "completed" assert [kwargs["aws_access_key_id"] for kwargs in bedrock_client_kwargs] == ["ASIABATCHSTATUSTAGGED"] + + +def test_cancel_batch_stops_and_polls_the_job_with_the_tagged_session(monkeypatch): + """Cancelling on a tag-gated role must forward the deployment's session tags to both the stop and status calls.""" + import litellm + + monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False) + monkeypatch.delenv("AWS_ROLE_ARN", raising=False) + tags = [{"Key": "team", "Value": "genai"}] + bedrock_client_kwargs: list[dict] = [] + fake_bedrock = MagicMock() + fake_bedrock.get_model_invocation_job.return_value = _fake_boto3_response(status="Stopped") + + def boto3_client(service_name, **kwargs): + if service_name == "sts": + return _TagGatedSTSClient(tags, "ASIABATCHCANCELTAGGED") + bedrock_client_kwargs.append(kwargs) + return fake_bedrock + + with patch("boto3.client", side_effect=boto3_client): + batch = litellm.cancel_batch( + batch_id=JOB_ARN, + custom_llm_provider="bedrock", + aws_access_key_id="AKIABATCHCANCELCALLER", + aws_secret_access_key="pod-caller-secret", + aws_role_name="arn:aws:iam::999999999999:role/litellm-batch-role", + aws_session_name="litellm-batch-session", + aws_session_tags=tags, + ) + + fake_bedrock.stop_model_invocation_job.assert_called_once_with(jobIdentifier=JOB_ARN) + assert batch.status == "cancelled" + assert [kwargs["aws_access_key_id"] for kwargs in bedrock_client_kwargs] == ["ASIABATCHCANCELTAGGED"] * 2 From cbeac276561c82e71f56de6d76e4d132170a2cc6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:12:13 -0700 Subject: [PATCH 232/411] fix(convert_dict_to_response): name the non-list choices type in the converter error When a provider returns choices as null, an object, a string or a number, the converter said the response had no 'choices' even though the key was present in the raw keys it listed. A shared message now keeps the old wording for a missing key and names the offending type otherwise. The cached-stream regression test also pins the chunk count so a leaked extra chunk fails it. --- .../convert_dict_to_response.py | 25 +++++++++++-------- .../test_convert_dict_to_response.py | 11 ++++---- .../test_streaming_handler.py | 1 + 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 29b75812bfb..87524d86c61 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -3,7 +3,7 @@ import json import re import time import traceback -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Final, Literal, cast import litellm @@ -151,6 +151,16 @@ def _clear_later_replay_slice_metadata(choice: StreamingChoices) -> None: del choice.enhancements +def _invalid_choices_message(response_object: Mapping[str, object]) -> str: + raw_keys: Final = list(response_object.keys()) + if "choices" not in response_object: + return f"LiteLLM: provider returned a response with no 'choices'. Raw keys: {raw_keys}" + return ( + f"LiteLLM: provider returned 'choices' that is not a list ({type(response_object['choices']).__name__}). " + f"Raw keys: {raw_keys}" + ) + + async def convert_to_streaming_response_async( response_object: dict | None = None, ): @@ -184,9 +194,7 @@ async def convert_to_streaming_response_async( raise APIError( status_code=500, - message=( - f"LiteLLM: provider returned a response with no 'choices'. Raw keys: {list(response_object.keys())}" - ), + message=_invalid_choices_message(response_object), llm_provider="", model="", ) @@ -292,9 +300,7 @@ def convert_to_streaming_response( raise APIError( status_code=500, - message=( - f"LiteLLM: provider returned a response with no 'choices'. Raw keys: {list(response_object.keys())}" - ), + message=_invalid_choices_message(response_object), llm_provider="", model="", ) @@ -628,10 +634,7 @@ def convert_to_model_response_object( raise APIError( status_code=500, - message=( - "LiteLLM: provider returned a response with no 'choices'. " - f"Raw keys: {list(response_object.keys())}" - ), + message=_invalid_choices_message(response_object), llm_provider="", model="", ) diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index c3e99f01cec..8e46ae21de6 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -167,9 +167,9 @@ def test_convert_missing_choices_raises_api_error() -> None: assert "no 'choices'" in str(exc_info.value) -@pytest.mark.parametrize("choices", [{}, "", None, 0]) +@pytest.mark.parametrize(("choices", "type_name"), [({}, "dict"), ("", "str"), (None, "NoneType"), (0, "int")]) @pytest.mark.asyncio -async def test_convert_non_list_choices_raises_api_error(choices: object) -> None: +async def test_convert_non_list_choices_raises_api_error(choices: object, type_name: str) -> None: from litellm.exceptions import APIError from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_streaming_response, @@ -183,14 +183,15 @@ async def test_convert_non_list_choices_raises_api_error(choices: object) -> Non "object": "chat.completion", "choices": choices, } - with pytest.raises(APIError, match="no 'choices'"): + expected: Final = f"'choices' that is not a list \\({type_name}\\)" + with pytest.raises(APIError, match=expected): convert_to_model_response_object( response_object=resp, model_response_object=ModelResponse(), response_type="completion", ) - with pytest.raises(APIError, match="no 'choices'"): + with pytest.raises(APIError, match=expected): list(convert_to_streaming_response(response_object=resp)) - with pytest.raises(APIError, match="no 'choices'"): + with pytest.raises(APIError, match=expected): async for _ in convert_to_streaming_response_async(response_object=resp): pass diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index ced7d6d677d..37e2031fdf4 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -2670,6 +2670,7 @@ async def test_cached_response_without_choices_streams_a_single_stop_chunk( chunks: Final = tuple([chunk async for chunk in wrapper]) + assert len(chunks) == 1 assert tuple(choice.finish_reason for chunk in chunks for choice in chunk.choices) == ("stop",) assert all(choice.delta.content in (None, "") for chunk in chunks for choice in chunk.choices) From 264fc82dc1bb66e840b625830c9d8be03d655a0b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:14:10 -0700 Subject: [PATCH 233/411] fix(rate_limiter): attach v3 priority rate limit headers on /v1/messages (#37228) * fix(rate_limiter): attach v3 priority rate limit headers on /v1/messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(playground): honor the Stream responses toggle for /v1/messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rate_limiter): drop explanatory docstrings from v3 dict response tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 18 ++--- .../hooks/parallel_request_limiter_v3.py | 33 +++------- .../add_retry_fallback_headers.py | 6 ++ .../hooks/test_dynamic_rate_limiter_v3.py | 57 ++++++++++++++++ .../hooks/test_parallel_request_limiter_v3.py | 65 ++++++++++++++++++ .../components/chat_ui/ChatUI.test.tsx | 53 +++++++++++++++ .../playground/components/chat_ui/ChatUI.tsx | 6 +- .../llm_calls/anthropic_messages.test.tsx | 66 ++++++++++++++++++- .../llm_calls/anthropic_messages.tsx | 33 +++++++--- 9 files changed, 291 insertions(+), 46 deletions(-) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index de8834449de..a074f02f4e8 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -32,6 +32,10 @@ from litellm.proxy.hooks.rate_limiter_utils import ( resolve_llm_provider_for_rate_limit, ) from litellm.proxy.utils import InternalUsageCache +from litellm.router_utils.add_retry_fallback_headers import ( + ensure_response_additional_headers, + response_has_hidden_params, +) from litellm.types.router import ModelGroupInfo from litellm.types.utils import CallTypesLiteral @@ -659,22 +663,12 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): data=data, user_api_key_dict=user_api_key_dict, response=response ) - # Add additional priority-specific headers - if isinstance(response, ModelResponse): + if response_has_hidden_params(response): priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) - - # Get existing additional headers - additional_headers: Final = getattr(response, "_hidden_params", {}).get("additional_headers", {}) or {} - - # Add priority information + additional_headers: Final = ensure_response_additional_headers(response) additional_headers["x-litellm-priority"] = priority or "default" additional_headers["x-litellm-rate-limiter-version"] = "v3" - # Update response - if not hasattr(response, "_hidden_params"): - response._hidden_params = {} - response._hidden_params["additional_headers"] = additional_headers - return response except Exception as e: diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 31437af7770..a72ae3bb1ea 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -52,6 +52,10 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit +from litellm.router_utils.add_retry_fallback_headers import ( + ensure_response_additional_headers, + response_has_hidden_params, +) from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage from litellm.types.utils import ( @@ -4677,34 +4681,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): Post-call hook to update rate limit headers in the response. """ try: - from pydantic import BaseModel - stash: Final = get_request_stash() litellm_proxy_rate_limit_response: Final = stash.rate_limit_response if stash is not None else None - if litellm_proxy_rate_limit_response is not None: - # Update response headers - if hasattr(response, "_hidden_params"): - _hidden_params = getattr(response, "_hidden_params") - else: - _hidden_params = None - - if _hidden_params is not None and ( - isinstance(_hidden_params, BaseModel) or isinstance(_hidden_params, dict) - ): - if isinstance(_hidden_params, BaseModel): - _hidden_params = _hidden_params.model_dump() - - _additional_headers: Final = self._merge_ratelimit_statuses_into_additional_headers( - additional_headers=_hidden_params.get("additional_headers", {}) or {}, + if litellm_proxy_rate_limit_response is not None and response_has_hidden_params(response): + additional_headers: Final = ensure_response_additional_headers(response) + additional_headers.update( + self._merge_ratelimit_statuses_into_additional_headers( + additional_headers={}, statuses=litellm_proxy_rate_limit_response["statuses"], ) - - setattr( - response, - "_hidden_params", - {**_hidden_params, "additional_headers": _additional_headers}, - ) + ) except Exception as e: verbose_proxy_logger.exception("Error in rate limit post-call hook: %s", e) diff --git a/litellm/router_utils/add_retry_fallback_headers.py b/litellm/router_utils/add_retry_fallback_headers.py index 3ec92ad226a..3251ea457cf 100644 --- a/litellm/router_utils/add_retry_fallback_headers.py +++ b/litellm/router_utils/add_retry_fallback_headers.py @@ -50,6 +50,12 @@ def prepare_response_for_header_attachment(response: object) -> object | None: return response +def response_has_hidden_params(response: object) -> bool: + if isinstance(response, dict): + return "_hidden_params" in response + return hasattr(response, "_hidden_params") + + def ensure_response_additional_headers(response: object) -> dict[str, object]: hidden_params: Final = get_hidden_params_dict(response, create=isinstance(response, dict)) _write_hidden_params(response, hidden_params) diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 0ff8b67b1a7..0cd6b4ede9c 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1861,3 +1861,60 @@ async def test_tpm_only_model_enforces_priority_and_model_capacity(monkeypatch): ) assert capacity_blocked.value.status_code == 429 assert "Model capacity reached" in capacity_blocked.value.detail["error"] + + +@pytest.mark.asyncio +async def test_post_call_success_hook_attaches_priority_headers_to_dict_response(): + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + RateLimitResponse, + RateLimitStatus, + get_or_create_request_stash, + ) + + handler = DynamicRateLimitHandler(internal_usage_cache=DualCache()) + get_or_create_request_stash().rate_limit_response = RateLimitResponse( + overall_code="OK", + statuses=[ + RateLimitStatus( + code="OK", + current_limit=75, + limit_remaining=74, + rate_limit_type="requests", + descriptor_key="priority_model", + ) + ], + ) + response = { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [], + "_hidden_params": {"additional_headers": {"x-litellm-attempted-retries": 0}}, + } + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(metadata={"priority": "premium"}), + response=response, + ) + + additional_headers = response["_hidden_params"]["additional_headers"] + assert additional_headers["x-litellm-attempted-retries"] == 0 + assert additional_headers["x-ratelimit-priority_model-limit-requests"] == 75 + assert additional_headers["x-ratelimit-priority_model-remaining-requests"] == 74 + assert additional_headers["x-litellm-priority"] == "premium" + assert additional_headers["x-litellm-rate-limiter-version"] == "v3" + + +@pytest.mark.asyncio +async def test_post_call_success_hook_leaves_raw_provider_dict_untouched(): + handler = DynamicRateLimitHandler(internal_usage_cache=DualCache()) + response = {"id": "msg_123", "type": "message", "role": "assistant", "content": []} + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(metadata={"priority": "premium"}), + response=response, + ) + + assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []} diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 4003286d887..6d382370f5f 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -6171,3 +6171,68 @@ async def test_success_hook_leaves_stash_untouched_for_non_batch_responses(): data={}, user_api_key_dict=user, response=ModelResponse(usage=Usage(total_tokens=5)) ) assert get_request_stash().batch_enqueued_reservation == reservation + + +@pytest.mark.asyncio +async def test_post_call_success_hook_attaches_ratelimit_headers_to_dict_response(): + from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitResponse, RateLimitStatus + + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + get_or_create_request_stash().rate_limit_response = RateLimitResponse( + overall_code="OK", + statuses=[ + RateLimitStatus( + code="OK", + current_limit=100, + limit_remaining=99, + rate_limit_type="requests", + descriptor_key="model_saturation_check", + ) + ], + ) + response = { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [], + "_hidden_params": {"additional_headers": {"x-litellm-attempted-retries": 0}}, + } + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(api_key=hash_token("sk-dict-response")), + response=response, + ) + + additional_headers = response["_hidden_params"]["additional_headers"] + assert additional_headers["x-litellm-attempted-retries"] == 0 + assert additional_headers["x-ratelimit-model_saturation_check-limit-requests"] == 100 + assert additional_headers["x-ratelimit-model_saturation_check-remaining-requests"] == 99 + + +@pytest.mark.asyncio +async def test_post_call_success_hook_leaves_raw_provider_dict_untouched(): + from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitResponse, RateLimitStatus + + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + get_or_create_request_stash().rate_limit_response = RateLimitResponse( + overall_code="OK", + statuses=[ + RateLimitStatus( + code="OK", + current_limit=100, + limit_remaining=99, + rate_limit_type="requests", + descriptor_key="model_saturation_check", + ) + ], + ) + response = {"id": "msg_123", "type": "message", "role": "assistant", "content": []} + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(api_key=hash_token("sk-raw-dict")), + response=response, + ) + + assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx index f07b66efdf8..bf6b092a2b0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import ChatUI from "./ChatUI"; import * as fetchModelsModule from "@/components/llm_calls/fetch_models"; import { makeOpenAIChatCompletionRequest } from "@/components/llm_calls/chat_completion"; +import { makeAnthropicMessagesRequest } from "../../llm_calls/anthropic_messages"; vi.mock("@/components/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn(), @@ -14,6 +15,10 @@ vi.mock("@/components/llm_calls/chat_completion", () => ({ makeOpenAIChatCompletionRequest: vi.fn().mockResolvedValue(undefined), })); +vi.mock("../../llm_calls/anthropic_messages", () => ({ + makeAnthropicMessagesRequest: vi.fn().mockResolvedValue(undefined), +})); + vi.mock("@/components/networking", () => ({ tagListCall: vi.fn().mockResolvedValue({}), vectorStoreListCall: vi.fn().mockResolvedValue({ data: [] }), @@ -32,6 +37,8 @@ beforeEach(() => { const CHAT_REQUEST_ARG_COUNT = 26; const STREAMING_ENABLED_ARG_INDEX = 25; +const MESSAGES_REQUEST_ARG_COUNT = 19; +const MESSAGES_STREAMING_ENABLED_ARG_INDEX = 18; async function openComboboxByPlaceholder(placeholder: string) { const user = userEvent.setup(); @@ -378,6 +385,52 @@ describe("ChatUI", () => { expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(false); }); + it("should send the /v1/messages request non-streaming after Stream responses is unchecked", async () => { + const user = userEvent.setup(); + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Test Key")).toBeInTheDocument(); + }); + + await selectComboboxOption("Select an endpoint", "/v1/messages"); + await selectComboboxOption("Select a Model", "Model 1"); + + await user.click(await screen.findByTestId("model-settings-button")); + + const streamingCheckbox = await screen.findByRole("checkbox", { name: /Stream responses/i }); + expect(streamingCheckbox).toBeChecked(); + await user.click(streamingCheckbox); + + await waitFor(() => { + expect(screen.getByRole("checkbox", { name: /Stream responses/i })).not.toBeChecked(); + }); + + const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)"); + await act(async () => { + fireEvent.change(messageInput, { target: { value: "hello" } }); + }); + await act(async () => { + fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" }); + }); + + await waitFor(() => { + expect(makeAnthropicMessagesRequest).toHaveBeenCalledTimes(1); + }); + + const requestArgs = vi.mocked(makeAnthropicMessagesRequest).mock.calls[0]; + expect(requestArgs).toHaveLength(MESSAGES_REQUEST_ARG_COUNT); + expect(requestArgs[MESSAGES_STREAMING_ENABLED_ARG_INDEX]).toBe(false); + }); + it("should force streaming in simplified mode even when the playground setting is off", async () => { sessionStorage.setItem("streamingEnabled", "false"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx index 35378d3d4e7..eca9e2323ae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx @@ -1025,6 +1025,7 @@ const ChatUI: React.FC = ({ mcpServers, mcpServerToolRestrictions, mcpToolsets, + streamingEnabled, ); } else if (endpointType === EndpointType.EMBEDDINGS) { await makeOpenAIEmbeddingsRequest( @@ -1174,7 +1175,10 @@ const ChatUI: React.FC = ({ return !model.mode || model.mode === "chat"; }; - const supportsStreamingToggle = endpointType === EndpointType.CHAT || endpointType === EndpointType.RESPONSES; + const supportsStreamingToggle = + endpointType === EndpointType.CHAT || + endpointType === EndpointType.RESPONSES || + endpointType === EndpointType.ANTHROPIC_MESSAGES; const modelsForEndpoint = useMemo( () => filterModelsForEndpoint(modelInfo, endpointType as EndpointType), [modelInfo, endpointType], diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx index 96ace129f87..9f030d8b2d4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx @@ -7,13 +7,27 @@ vi.mock("@/components/networking", () => ({ })); const mockMessagesStream = vi.fn(); +const mockMessagesCreate = vi.fn(); vi.mock("@anthropic-ai/sdk", () => ({ default: vi.fn(function () { - return { messages: { stream: mockMessagesStream } }; + return { messages: { stream: mockMessagesStream, create: mockMessagesCreate } }; }), })); +const NON_STREAMING_ARGS = [ + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // mcpToolsets + false, // streamingEnabled +] as const; + describe("anthropic_messages prompt cache usage", () => { const captureUsage = async (usage: Record): Promise => { async function* mockStream() { @@ -59,3 +73,53 @@ describe("anthropic_messages prompt cache usage", () => { expect(usageData.promptTokens).toBe(5000); }); }); + +describe("anthropic_messages non-streaming", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("sends stream:false through messages.create and renders the full reply at once", async () => { + mockMessagesCreate.mockResolvedValue({ + content: [ + { type: "thinking", thinking: "considering" }, + { type: "text", text: "OK" }, + ], + usage: { input_tokens: 12, output_tokens: 3, cache_read_input_tokens: 7 }, + }); + const updateTextUI = vi.fn(); + const onReasoningContent = vi.fn(); + const onUsageData = vi.fn(); + + await makeAnthropicMessagesRequest( + [{ role: "user", content: "Hello" }], + updateTextUI, + "claude-haiku-4-5", + "test-token", + undefined, + undefined, + onReasoningContent, + undefined, + onUsageData, + ...NON_STREAMING_ARGS, + ); + + expect(mockMessagesStream).not.toHaveBeenCalled(); + expect(mockMessagesCreate).toHaveBeenCalledTimes(1); + expect(mockMessagesCreate.mock.calls[0][0]).toMatchObject({ model: "claude-haiku-4-5", stream: false }); + expect(updateTextUI).toHaveBeenCalledWith("assistant", "OK", "claude-haiku-4-5"); + expect(onReasoningContent).toHaveBeenCalledWith("considering"); + const expectedUsage: TokenUsage = { completionTokens: 3, promptTokens: 12, totalTokens: 15, cacheReadTokens: 7 }; + expect(onUsageData).toHaveBeenCalledWith(expectedUsage); + }); + + it("keeps streaming as the default when the flag is omitted", async () => { + async function* emptyStream() {} + mockMessagesStream.mockReturnValue(emptyStream()); + + await makeAnthropicMessagesRequest([{ role: "user", content: "Hello" }], vi.fn(), "claude-haiku-4-5", "test-token"); + + expect(mockMessagesCreate).not.toHaveBeenCalled(); + expect(mockMessagesStream.mock.calls[0][0]).toMatchObject({ stream: true }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx index 14afa013768..9dd2f675c44 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx @@ -7,6 +7,13 @@ import { getProxyBaseUrl } from "@/components/networking"; import { toast } from "@/lib/toast"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; +const toTokenUsage = (usage: Anthropic.Usage): TokenUsage => ({ + completionTokens: usage.output_tokens, + promptTokens: usage.input_tokens, + totalTokens: usage.input_tokens + usage.output_tokens, + ...extractPromptCacheTokens(usage), +}); + export async function makeAnthropicMessagesRequest( messages: MessageType[], updateTextUI: (role: string, delta: string, model?: string) => void, @@ -26,6 +33,7 @@ export async function makeAnthropicMessagesRequest( mcpServers?: MCPServer[], mcpServerToolRestrictions?: Record, mcpToolsets?: MCPToolset[], + streamingEnabled: boolean = true, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -58,7 +66,7 @@ export async function makeAnthropicMessagesRequest( const requestBody: any = { model: selectedModel, messages: messages.map((m) => ({ role: m.role, content: m.content })), - stream: true, + stream: streamingEnabled, max_tokens: 1024, // @ts-ignore - litellm specific parameter litellm_trace_id: traceId, @@ -74,6 +82,20 @@ export async function makeAnthropicMessagesRequest( if (vector_store_ids) requestBody.vector_store_ids = vector_store_ids; if (guardrails) requestBody.guardrails = guardrails; if (policies) requestBody.policies = policies; + + if (!streamingEnabled) { + const message: Anthropic.Message = await client.messages.create({ ...requestBody, stream: false }, { signal }); + for (const block of message.content) { + if (block.type === "text") { + updateTextUI("assistant", block.text, selectedModel); + } else if (block.type === "thinking" && onReasoningContent) { + onReasoningContent(block.thinking); + } + } + onUsageData?.(toTokenUsage(message.usage)); + return; + } + // Use the streaming helper method for cleaner async iteration // @ts-ignore - The SDK types might not include all litellm-specific parameters const stream = client.messages.stream(requestBody, { signal }); @@ -105,14 +127,7 @@ export async function makeAnthropicMessagesRequest( // Process usage data from message_delta events if (messageStreamEvent.type === "message_delta" && (messageStreamEvent as any).usage && onUsageData) { - const usage = (messageStreamEvent as any).usage; - const usageData: TokenUsage = { - completionTokens: usage.output_tokens, - promptTokens: usage.input_tokens, - totalTokens: usage.input_tokens + usage.output_tokens, - ...extractPromptCacheTokens(usage), - }; - onUsageData(usageData); + onUsageData(toTokenUsage((messageStreamEvent as any).usage)); } } } catch (error) { From ae6893bc6498d45785fad095821813d13e21cce3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:38:55 -0700 Subject: [PATCH 234/411] refactor(guardrails): narrow the Responses tool-call item mapping types --- .../openai/responses/guardrail_translation/handler.py | 4 ++-- .../responses/test_openai_responses_guardrail_handler.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index e78e1f56915..fd0d7b66bee 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -193,11 +193,11 @@ def _is_tool_call_item(item: object) -> bool: return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES -def _tool_call_output_item_mapping(item: object) -> Mapping[str, Any] | None: +def _tool_call_output_item_mapping(item: object) -> Mapping[str, object] | None: if stream_item_field(item, "type") not in _TOOL_CALL_ITEM_TYPES: return None if isinstance(item, Mapping): - return cast("Mapping[str, Any]", item) # cast-ok: output items are str-keyed JSON objects + return cast("Mapping[str, object]", item) # cast-ok: output items are str-keyed JSON objects return item.model_dump() if isinstance(item, BaseModel) else None diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 628ada05267..cf4895f637f 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -14,6 +14,7 @@ import pytest from fastapi import HTTPException +from pydantic import BaseModel from openai.types.responses import ( ResponseCustomToolCall, ResponseCustomToolCallInputDeltaEvent, @@ -28,6 +29,7 @@ from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.types.llms.openai import ChatCompletionToolCallChunk from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -68,7 +70,7 @@ class PersimmonMaskingGuardrail(CustomGuardrail): inputs: GenericGuardrailAPIInputs, request_data: dict, input_type: Literal["request", "response"], - logging_obj: Optional[Any] = None, + logging_obj: Optional[LiteLLMLoggingObj] = None, ) -> GenericGuardrailAPIInputs: tool_calls = [ { @@ -593,7 +595,7 @@ class TestOpenAIResponsesHandlerToolCallExtraction: texts_to_check: List[str] = [] images_to_check: List[str] = [] - tool_calls_to_check: List[Any] = [] + tool_calls_to_check: List[ChatCompletionToolCallChunk] = [] task_mappings: List[Tuple[int, int]] = [] # Extract tool calls @@ -1476,7 +1478,7 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: ) handler = OpenAIResponsesHandler() - typed_events: List[Any] = [ + typed_events: List[BaseModel] = [ model.model_validate({**event, "sequence_number": sequence_number}) for sequence_number, (model, event) in enumerate( zip( From eee239cd9e6b3b3e4bc3d3a515fa7d5160b51df9 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 9 Sep 2026 15:20:58 -0700 Subject: [PATCH 235/411] fix(jwt): cascade-delete JWT key mappings when their virtual key is deleted LiteLLM_JWTKeyMapping_token_fkey was created ON DELETE RESTRICT, so deleting a virtual key that a JWT mapping pointed at failed with a foreign key violation on every deletion path (/key/delete, Admin UI, alias delete, team and user cascades). Declaring onDelete: Cascade on the relation lets the database clean the mapping up uniformly, so the next JWT call from that identity re-registers against the newly created key. Rebase of #33703 onto current staging. Claude-Session: https://claude.ai/code/session_011Tn3657NkV6ojLqewL64Kb --- .../migration.sql | 15 +++++ .../litellm_proxy_extras/schema.prisma | 2 +- litellm/proxy/schema.prisma | 2 +- schema.prisma | 2 +- .../test_litellm_proxy_extras_utils.py | 67 +++++++++++++++++++ 5 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_cascade_delete_jwt_key_mapping_on_token_delete/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_cascade_delete_jwt_key_mapping_on_token_delete/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_cascade_delete_jwt_key_mapping_on_token_delete/migration.sql new file mode 100644 index 00000000000..e5d48abcb52 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_cascade_delete_jwt_key_mapping_on_token_delete/migration.sql @@ -0,0 +1,15 @@ +-- DropForeignKey +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_JWTKeyMapping_token_fkey') THEN + ALTER TABLE "LiteLLM_JWTKeyMapping" DROP CONSTRAINT "LiteLLM_JWTKeyMapping_token_fkey"; + END IF; +END $$; + +-- AddForeignKey +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_JWTKeyMapping_token_fkey') THEN + ALTER TABLE "LiteLLM_JWTKeyMapping" ADD CONSTRAINT "LiteLLM_JWTKeyMapping_token_fkey" FOREIGN KEY ("token") REFERENCES "LiteLLM_VerificationToken"("token") ON DELETE CASCADE ON UPDATE CASCADE; + END IF; +END $$; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 3d254cd2ea2..05c5aad9303 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -492,7 +492,7 @@ model LiteLLM_JWTKeyMapping { updated_at DateTime @default(now()) @updatedAt updated_by String? - litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token]) + litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade) @@unique([jwt_claim_name, jwt_claim_value]) @@index([jwt_claim_name, jwt_claim_value, is_active]) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 3d254cd2ea2..05c5aad9303 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -492,7 +492,7 @@ model LiteLLM_JWTKeyMapping { updated_at DateTime @default(now()) @updatedAt updated_by String? - litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token]) + litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade) @@unique([jwt_claim_name, jwt_claim_value]) @@index([jwt_claim_name, jwt_claim_value, is_active]) diff --git a/schema.prisma b/schema.prisma index 3d254cd2ea2..05c5aad9303 100644 --- a/schema.prisma +++ b/schema.prisma @@ -492,7 +492,7 @@ model LiteLLM_JWTKeyMapping { updated_at DateTime @default(now()) @updatedAt updated_by String? - litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token]) + litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade) @@unique([jwt_claim_name, jwt_claim_value]) @@index([jwt_claim_name, jwt_claim_value, is_active]) diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 3fab20a28ad..e5826b18668 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -2,6 +2,7 @@ import glob import os import re import sys +from pathlib import Path import pytest @@ -870,3 +871,69 @@ class TestMigrateDeployAttemptAccounting: harness.run() assert len(harness.deploy_calls) == 1 assert harness.resolved == [] + + +class TestJWTKeyMappingCascade: + """Regression tests for issue #33702. + + A virtual key referenced by a LiteLLM_JWTKeyMapping row could not be deleted + because LiteLLM_JWTKeyMapping_token_fkey was created ON DELETE RESTRICT, so + deleting the key (Admin UI, /key/delete, team delete, ...) raised a foreign + key violation. The mapping must be removed automatically when its key is + deleted, which the FK now enforces via ON DELETE CASCADE. + """ + + _FK_NAME = "LiteLLM_JWTKeyMapping_token_fkey" + + def _effective_on_delete(self): + """Replay every migration in order and return the last ON DELETE action + declared for the JWT key mapping FK.""" + action = None + for _migration_name, sql in _get_all_migrations(): + for match in re.finditer( + rf'ADD\s+CONSTRAINT\s+"{re.escape(self._FK_NAME)}".*?' + r"ON\s+DELETE\s+(CASCADE|RESTRICT|SET\s+NULL|NO\s+ACTION|SET\s+DEFAULT)", + sql, + re.IGNORECASE | re.DOTALL, + ): + action = re.sub(r"\s+", " ", match.group(1).upper()) + return action + + def test_fk_effective_on_delete_is_cascade(self): + """The final FK definition across all migrations must cascade deletes.""" + assert self._effective_on_delete() == "CASCADE", ( + f"{self._FK_NAME} must end up ON DELETE CASCADE so deleting a " + "virtual key removes its JWT key mapping (issue #33702)" + ) + + def test_schema_declares_cascade_on_relation(self): + """schema.prisma must declare onDelete: Cascade on the mapping relation + so the generated client and DB agree.""" + schema_paths = glob.glob( + os.path.abspath( + os.path.join( + os.path.dirname(__file__), "../../**/schema.prisma" + ) + ), + recursive=True, + ) + declaring = tuple( + (path, schema) + for path, schema in ((p, Path(p).read_text()) for p in schema_paths) + if "model LiteLLM_JWTKeyMapping" in schema + ) + assert declaring, "No schema.prisma declaring LiteLLM_JWTKeyMapping found" + for path, schema in declaring: + match = re.search( + r"litellm_verification_token\s+LiteLLM_VerificationToken\s+@relation\(([^)]*)\)", + schema, + ) + assert match is not None, ( + f"{path} declares LiteLLM_JWTKeyMapping but its verification token " + "relation could not be parsed, so this test cannot vouch for it " + "(issue #33702)" + ) + assert "onDelete: Cascade" in match.group(1), ( + f"{path} must declare onDelete: Cascade on the JWT key mapping " + "relation (issue #33702)" + ) From 6e96885629521b69c01c56faa165300963f6fb9a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:40:15 -0700 Subject: [PATCH 236/411] fix(proxy): accept non-string callback vars in default_team_settings (#40458) * fix(proxy): accept non-string callback vars in default_team_settings A YAML boolean such as turn_off_message_logging: true in a default_team_settings block failed TeamCallbackMetadata's str-only callback_vars validation and errored the request before any callback ran. Stringify the value the same way AddTeamCallback does. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): drop docstring from default_team_settings bool regression test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): move default_team_settings bool regression test to mapped pre_call_utils suite Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/litellm_pre_call_utils.py | 4 ++- .../proxy/test_litellm_pre_call_utils.py | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 2925226f235..da033dc2276 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1760,7 +1760,9 @@ class LiteLLMProxyRequestSetup: callback_vars_dict.pop("success_callback", None) callback_vars_dict.pop("failure_callback", None) callback_vars_dict = { - key: (litellm.utils.get_secret(value, default_value=value) or value if isinstance(value, str) else value) + key: ( + litellm.utils.get_secret(value, default_value=value) or value if isinstance(value, str) else str(value) + ) for key, value in callback_vars_dict.items() } diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 7564fa91c43..94aaa32519e 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -35,6 +35,7 @@ from litellm.proxy.litellm_pre_call_utils import ( ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY +from litellm.litellm_core_utils.redact_messages import _get_turn_off_message_logging_from_dynamic_params from litellm.litellm_core_utils.get_provider_specific_headers import ( ProviderSpecificHeaderUtils, ) @@ -7909,3 +7910,36 @@ async def test_client_supplied_omit_marker_never_reaches_the_spend_log( if general_settings.get("missing_session_id") == "generate" else "per-call-random-trace-id" ) + + +def test_default_team_settings_bool_turn_off_message_logging_redacts(): + from litellm.proxy.proxy_server import ProxyConfig + + pc = ProxyConfig() + pc.config = { + "litellm_settings": { + "default_team_settings": [ + { + "team_id": "team-redact", + "success_callback": ["gcs_bucket"], + "failure_callback": ["gcs_bucket"], + "turn_off_message_logging": True, + } + ] + } + } + + callback_metadata = LiteLLMProxyRequestSetup.add_team_based_callbacks_from_config( + team_id="team-redact", + proxy_config=pc, + ) + + assert callback_metadata is not None + assert callback_metadata.success_callback == ["gcs_bucket"] + assert callback_metadata.callback_vars == {"turn_off_message_logging": "True"} + assert ( + _get_turn_off_message_logging_from_dynamic_params( + {"standard_callback_dynamic_params": dict(callback_metadata.callback_vars)} + ) + is True + ) From f5a410585af9fe2c06f0e5fa07ceddb2e0be46e0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:44:29 -0700 Subject: [PATCH 237/411] fix(responses): keep a custom tool_choice type in the bridged echo and type the new tests --- .../transformation.py | 23 +++++++++++++------ .../test_litellm_completion_responses.py | 17 +++++++++----- .../test_streaming_iterator_transformation.py | 9 ++++---- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 878f493b58e..6b9931b7df2 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -27,9 +27,10 @@ from openai.types.chat.chat_completion_named_tool_choice_param import ( ) from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.response_create_params import ResponseInputParam +from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger @@ -272,13 +273,21 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_tool_choice_for_responses_api_response(tool_choice: object) -> ToolChoice: - normalized: Final = LiteLLMCompletionResponsesConfig._transform_tool_choice(tool_choice) - match normalized: - case None: - return "auto" - case {"type": "function", "function": {"name": str(function_name)}}: + if tool_choice is None: + return "auto" + try: + return _RESPONSES_API_TOOL_CHOICE_ADAPTER.validate_python(tool_choice) + except ValidationError: + return LiteLLMCompletionResponsesConfig._chat_tool_choice_as_responses_api_tool_choice(tool_choice) + + @staticmethod + def _chat_tool_choice_as_responses_api_tool_choice(tool_choice: object) -> ToolChoice: + match tool_choice, LiteLLMCompletionResponsesConfig._transform_tool_choice(tool_choice): + case {"type": "custom"}, {"function": {"name": str(custom_name)}}: + return ToolChoiceCustomParam(type="custom", name=custom_name) + case _, {"type": "function", "function": {"name": str(function_name)}}: return ToolChoiceFunctionParam(type="function", name=function_name) - case _: + case _, normalized: return _RESPONSES_API_TOOL_CHOICE_ADAPTER.validate_python(normalized) @staticmethod diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2aab660b5b7..8f2937cb252 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1,4 +1,5 @@ import json +from typing import Final import pytest @@ -1426,7 +1427,9 @@ class TestToolChoiceTransformation: [ ({"type": "function", "name": "run_command"}, {"type": "function", "name": "run_command"}), ({"type": "function", "function": {"name": "run_command"}}, {"type": "function", "name": "run_command"}), - ({"type": "custom", "name": "ApplyPatch"}, {"type": "function", "name": "ApplyPatch"}), + ({"type": "custom", "name": "ApplyPatch"}, {"type": "custom", "name": "ApplyPatch"}), + ({"type": "custom", "custom": {"name": "ApplyPatch"}}, {"type": "custom", "name": "ApplyPatch"}), + ({"type": "function"}, "required"), ({"type": "tool"}, "required"), ({"type": "auto"}, "auto"), ("required", "required"), @@ -1434,14 +1437,16 @@ class TestToolChoiceTransformation: (None, "auto"), ], ) - def test_transform_tool_choice_for_responses_api_response(self, request_tool_choice, expected): - result = LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( + def test_transform_tool_choice_for_responses_api_response( + self, request_tool_choice: object, expected: str | dict[str, str] + ) -> None: + result: Final = LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( request_tool_choice ) assert result == expected - def test_non_streamed_response_echoes_named_tool_choice_in_responses_api_shape(self): - chat_completion_response = ModelResponse( + def test_non_streamed_response_echoes_named_tool_choice_in_responses_api_shape(self) -> None: + chat_completion_response: Final = ModelResponse( id="chatcmpl-named-tool-choice", created=1748575031, model="claude-haiku-4-5", @@ -1465,7 +1470,7 @@ class TestToolChoiceTransformation: ], ) - responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + responses_api_response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( request_input="Run the command pwd.", responses_api_request={"tool_choice": {"type": "function", "name": "run_command"}}, chat_completion_response=chat_completion_response, diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index d4b565f82a1..29061db97dc 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -11,6 +11,7 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve """ import json +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -657,8 +658,8 @@ def _tool_call_chunk(finish_reason: str | None = None) -> ModelResponseStream: ) -def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape(): - iterator = LiteLLMCompletionStreamingIterator( +def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape() -> None: + iterator: Final = LiteLLMCompletionStreamingIterator( model="claude-haiku-4-5", litellm_custom_stream_wrapper=_FakeStreamWrapper([_tool_call_chunk(finish_reason="tool_calls")]), request_input="Run the command pwd.", @@ -670,9 +671,9 @@ def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape(): litellm_metadata={}, ) - events = list(iterator) + events: Final = list(iterator) - response_events = [event for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] + response_events: Final = [event for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] assert [event.type for event in response_events] == [ "response.created", "response.in_progress", From eb45a088d3d3c255118d0effb41e4613af0907c0 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 15:53:05 -0700 Subject: [PATCH 238/411] fix(router): resolve team-scoped auto-routers by their public name (#40432) A team-scoped auto-router is stored under an internal model_name_{team_id}_{uuid} with the caller-facing name in model_info.team_public_model_name, and the four pre-routing strategy registries key on that internal name. A team key asks for the public name, so the strategy lookup missed, the team early-resolve exit handed back the marker deployment itself, and every call 400'd with "Unmapped LLM provider". The strategy lookup now resolves the requested name through the same team-first, then global, then admin-across-teams deployment resolution the deployment path uses, and looks the registries up under the model_name of whatever that resolves to. Both exits of _common_checks_available_deployment drop strategy markers through one helper, so a marker-only resolution is rejected as uncallable on every path. The request team id has one reader. Resolves LIT-7363 Claude-Session: https://claude.ai/code/session_01NU97S7d2FUDDvTk59k53Wp Co-authored-by: Claude Fable 5.1 --- .../guardrails/auto_router_compression.py | 25 +- litellm/router.py | 144 +++++---- litellm/router_utils/common_utils.py | 14 +- .../test_auto_router_compression.py | 26 +- .../proxy/test_common_request_processing.py | 2 +- .../router_strategy/test_complexity_router.py | 4 +- tests/test_litellm/test_router.py | 301 +++++++++++++++++- 7 files changed, 417 insertions(+), 99 deletions(-) diff --git a/litellm/proxy/guardrails/auto_router_compression.py b/litellm/proxy/guardrails/auto_router_compression.py index 98707e7ddca..c37b9fff1f0 100644 --- a/litellm/proxy/guardrails/auto_router_compression.py +++ b/litellm/proxy/guardrails/auto_router_compression.py @@ -80,17 +80,19 @@ def policy_from_litellm_params(litellm_params: Mapping[str, object]) -> AutoRout def policy_for_model( llm_router: "Router | None", model_alias: str, - team_id: str | None, + request_kwargs: Mapping[str, object], request_tags: Sequence[str], ) -> AutoRouterCompressionPolicy | None: - """The compression policy of the auto router marker `model_alias` resolves to. + """The compression policy of the auto router marker `model_alias` resolves to for this caller. - Pre-call arming and the routing hook both resolve through here, so an alias with - several tag-scoped markers cannot suppress under one and then route under another. + Pre-call arming and the routing hook both resolve through here, and here resolves through the + router's own request-scoped deployment lookup, so an alias with several tag-scoped markers + cannot suppress under one and then route under another, and a team router reached by its + public name carries its policy for every principal that can reach it. """ if llm_router is None: return None - deployments: Final = llm_router.get_model_list(model_name=model_alias, team_id=team_id) or () + deployments: Final = llm_router.deployments_for_request(model_alias, request_kwargs) markers: Final = tuple( litellm_params for deployment in deployments @@ -108,17 +110,6 @@ def policy_for_model( return next((policy for policy in candidates if policy is not None), None) -def team_id_from_request(request_kwargs: Mapping[str, object]) -> str | None: - """The caller's team id, from whichever metadata bucket this surface writes to.""" - for meta_key in ("metadata", "litellm_metadata"): - meta = request_kwargs.get(meta_key) - if isinstance(meta, Mapping): - team_id = meta.get("user_api_key_team_id") - if isinstance(team_id, str): - return team_id - return None - - def _compression_guardrail_classes() -> tuple[type, ...]: """The registered guardrail classes whose provider compresses prompts.""" from litellm.proxy.guardrails.guardrail_registry import guardrail_class_registry @@ -172,7 +163,7 @@ async def arm_pre_call( policy: Final = policy_for_model( llm_router=llm_router, model_alias=model_alias, - team_id=team_id_from_request(data), + request_kwargs=data, request_tags=_get_tags_from_request_kwargs(data), ) if policy is None: diff --git a/litellm/router.py b/litellm/router.py index 68ace283949..88601e5b97b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -148,6 +148,7 @@ from litellm.router_utils.common_utils import ( _is_proxy_admin_request, filter_team_based_models, filter_web_search_deployments, + get_request_team_id, resolve_model_group_alias, truncate_fallback_error_detail, warn_on_provider_credential_mismatch, @@ -12337,27 +12338,7 @@ class Router: if team_deployments: return model, team_deployments elif include_team_models: - team_deployments = [ - self.model_list[index] - for (_, public_model_name), indices in self.team_model_to_deployment_indices.items() - if public_model_name == model - for index in indices - ] - team_ids: Final = { - team_id - for deployment in team_deployments - for team_id in [(deployment.get("model_info") or {}).get("team_id")] - if team_id is not None - } - if len(team_ids) > 1: - raise litellm.BadRequestError( - message=( - f"Model name '{model}' matches deployments from multiple teams. " - "Specify the deployment ID directly to disambiguate." - ), - model=model, - llm_provider="", - ) + team_deployments = self._team_deployments_across_teams(model) if team_deployments: return model, team_deployments @@ -12384,6 +12365,45 @@ class Router: return None + def _team_deployments_across_teams(self, model: str) -> list[DeploymentTypedDict]: + """Every team's deployments under public name `model`, for a proxy admin calling without a team.""" + team_deployments: Final = [ + self.model_list[index] + for (_, public_model_name), indices in self.team_model_to_deployment_indices.items() + if public_model_name == model + for index in indices + ] + team_ids: Final = { + team_id + for deployment in team_deployments + for team_id in [(deployment.get("model_info") or {}).get("team_id")] + if team_id is not None + } + if len(team_ids) > 1: + raise litellm.BadRequestError( + message=( + f"Model name '{model}' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." + ), + model=model, + llm_provider="", + ) + return team_deployments + + def deployments_for_request( + self, model: str, request_kwargs: Mapping[str, object] + ) -> Sequence[DeploymentTypedDict]: + """The deployments `model` names for this caller, through the same alias, then team-first, then + global, then admin-across-teams resolution `_common_checks_available_deployment` applies, so + strategy selection and compression policy can never disagree with deployment selection about + which marker a name means.""" + registered_name: Final = self._get_model_from_alias(model=model) or model + team_id: Final = get_request_team_id(request_kwargs) + deployments: Final = self._get_all_deployments(model_name=registered_name, team_id=team_id) + if deployments or team_id is not None or not _is_proxy_admin_request(request_kwargs): + return deployments + return self._team_deployments_across_teams(registered_name) + @staticmethod def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool: litellm_params: Final = deployment.get("litellm_params") @@ -12411,11 +12431,7 @@ class Router: - Dict, if specific model chosen """ - request_team_id: str | None = None - if request_kwargs is not None: - metadata: Final = request_kwargs.get("metadata") or {} - litellm_metadata: Final = request_kwargs.get("litellm_metadata") or {} - request_team_id = metadata.get("user_api_key_team_id") or litellm_metadata.get("user_api_key_team_id") + request_team_id: Final = get_request_team_id(request_kwargs) # check if aliases set on litellm model alias map if specific_deployment is True: return model, self._get_deployment_by_litellm_model(model=model) @@ -12440,7 +12456,9 @@ class Router: include_team_models=_is_proxy_admin_request(request_kwargs), ) if early is not None: - return early + if not isinstance(early[1], list): + return early + return early[0], self._drop_strategy_markers(early[0], early[1]) ## get healthy deployments ### get all deployments @@ -12517,19 +12535,22 @@ class Router: model ] # update the model to the actual value if an alias has been passed in - marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments) - if not any(marker_flags): - return model, healthy_deployments - selectable: Final = [ # mutable-ok: matches this function's list contract expected by downstream filters - d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker + return model, self._drop_strategy_markers(model, healthy_deployments) + + def _drop_strategy_markers( + self, model: str, deployments: Sequence[DeploymentTypedDict] + ) -> list[DeploymentTypedDict]: + """A strategy marker is never a callable deployment, whichever resolution arm produced it.""" + selectable: Final = [ # mutable-ok: matches _common_checks_available_deployment's list contract + d for d in deployments if not self._is_strategy_marker_deployment(d) ] - if not selectable: + if deployments and not selectable: raise litellm.BadRequestError( message=f"You passed in model={model}. {RouterErrors.only_strategy_marker_deployments.value}", model=model, llm_provider="", ) - return model, selectable + return selectable def _filter_deployments_by_model_access_groups( self, @@ -13219,12 +13240,8 @@ class Router: return filtered - def _model_name_has_plain_deployments(self, model: str) -> bool: - indices: Final = self.model_name_to_deployment_indices.get(model) or () - return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices) - def _select_pre_routing_strategy( - self, model: str, request_kwargs: dict + self, model: str, request_kwargs: Mapping[str, object] ) -> "TaggedPreRoutingStrategy[PreRoutingStrategy] | None": """ Resolve the pre-routing strategy for `model`, disambiguating deployments @@ -13235,6 +13252,12 @@ class Router: deployment the strategy was registered from via its (model_name, tags) pair. + The registries are keyed by the marker deployment's own `model_name`, which + for a team-scoped router is the internal `model_name_{team}_{uuid}` while + the caller sends the team's public name. So the names looked up are the + `model_name`s of whatever deployments this caller's request resolves `model` + to, and `model` itself when it resolves to none. + With tag filtering enabled, router-wide or by the request's enable_tag_filtering (which the proxy sets from key/team router_settings), strategies that all carry real tags matching none of @@ -13242,12 +13265,14 @@ class Router: deployments: returning None hands the request to ordinary tag-aware deployment selection. """ - candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [ - *self.auto_routers.get(model, []), - *self.complexity_routers.get(model, []), - *self.adaptive_routers.get(model, []), - *self.quality_routers.get(model, []), - ] + registries: Final = (self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers) + if not any(registries): + return None + deployments: Final = self.deployments_for_request(model, request_kwargs) + registered_names: Final = tuple(dict.fromkeys(str(d["model_name"]) for d in deployments)) or (model,) + candidates: Final = tuple( + tagged for registry in registries for name in registered_names for tagged in registry.get(name, []) + ) if not candidates: return None @@ -13265,7 +13290,7 @@ class Router: if ( (self.enable_tag_filtering or request_scoped_filtering) and all(tagged.tags for tagged in candidates) - and self._model_name_has_plain_deployments(model) + and any(not self._is_strategy_marker_deployment(d) for d in deployments) ): return None return candidates[0] @@ -13377,11 +13402,12 @@ class Router: Used for the litellm auto-router to modify the request before the routing decision is made. - `model` is whatever the caller asked for, which may be a `model_group_alias` key, while the - strategy registries and the marker deployment are keyed by the marker's own `model_name`, so - every lookup below resolves the alias first. Only the lookups: the caller-facing name stays - the alias, since spend metadata is stamped before routing and the response carries the tier - group the strategy picked. + `model` is whatever the caller asked for, which may be a `model_group_alias` key or a team's + public model name, while the strategy registries and the marker deployment are keyed by the + marker's own `model_name`, so every lookup below resolves the alias first and the team name + through the deployment path. Only the lookups: the caller-facing name stays the alias, since + spend metadata is stamped before routing and the response carries the tier group the + strategy picked. """ requested_registered_model_name: Final = self._get_model_from_alias(model=model) or model registered_model_name: Final = await self._resolve_claude_code_session_router( @@ -13418,7 +13444,6 @@ class Router: messages_for_routing, model_hop_compression_armed, policy_for_model, - team_id_from_request, ) # Same tag-aware lookup the proxy's pre-call arming used, so an alias with @@ -13426,7 +13451,7 @@ class Router: compression_policy: Final = policy_for_model( llm_router=self, model_alias=registered_model_name, - team_id=team_id_from_request(request_kwargs), + request_kwargs=request_kwargs, request_tags=_get_tags_from_request_kwargs(request_kwargs), ) # Shared compression already ran in the pre-call hook, so reuse it rather than @@ -13495,7 +13520,9 @@ class Router: # Per-tier `litellm_params` on the hook response are deliberate overrides # the caller applies on top, so those keys are never forwarded here. marker_params: Final = ( - self._forwardable_alias_marker_params(model=registered_model_name, strategy_tags=selected_strategy.tags) + self._forwardable_alias_marker_params( + model=registered_model_name, strategy_tags=selected_strategy.tags, request_kwargs=request_kwargs + ) if pre_routing_hook_response is not None else () ) @@ -13513,13 +13540,14 @@ class Router: return pre_routing_hook_response def _forwardable_alias_marker_params( - self, model: str, strategy_tags: tuple[str, ...] + self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object] ) -> tuple[tuple[str, object], ...]: marker_params: Final = tuple( litellm_params - for idx in self.model_name_to_deployment_indices.get(model, ()) - if isinstance(litellm_params := self.model_list[idx].get("litellm_params", {}), dict) - and str(litellm_params.get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX) + for deployment in self.deployments_for_request(model, request_kwargs) + if str((litellm_params := deployment["litellm_params"]).get("model", "")).startswith( + AUTO_ROUTER_MODEL_PREFIX + ) ) tag_matched: Final = tuple( params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 280a7defcf8..c1c6d25beca 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -26,6 +26,18 @@ def _is_proxy_admin_request(request_kwargs: Mapping[str, object] | None) -> bool return getattr(user_api_key_auth, "user_role", None) == "proxy_admin" +def get_request_team_id(request_kwargs: Mapping[str, object] | None) -> str | None: + """The caller's team id, from whichever metadata bucket this surface writes to.""" + if request_kwargs is None: + return None + for bucket_name in ("metadata", "litellm_metadata"): + bucket = request_kwargs.get(bucket_name) + team_id = bucket.get("user_api_key_team_id") if isinstance(bucket, Mapping) else None + if isinstance(team_id, str) and team_id: + return team_id + return None + + def resolve_model_group_alias(model_group_alias: object, model: str) -> str | None: """ Resolve ``model`` through a ``model_group_alias`` map. @@ -110,7 +122,7 @@ def filter_team_based_models( metadata: Final = request_kwargs.get("metadata") or {} litellm_metadata: Final = request_kwargs.get("litellm_metadata") or {} - request_team_id: Final = metadata.get("user_api_key_team_id") or litellm_metadata.get("user_api_key_team_id") + request_team_id: Final = get_request_team_id(request_kwargs) if request_team_id is None and _is_proxy_admin_request(request_kwargs) and isinstance(healthy_deployments, list): requested_model: Final = ( request_kwargs.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group") diff --git a/tests/test_litellm/proxy/guardrails/test_auto_router_compression.py b/tests/test_litellm/proxy/guardrails/test_auto_router_compression.py index db2f94306fb..0d723d6671f 100644 --- a/tests/test_litellm/proxy/guardrails/test_auto_router_compression.py +++ b/tests/test_litellm/proxy/guardrails/test_auto_router_compression.py @@ -56,13 +56,13 @@ class TestPolicyFromLitellmParams: class _FakeRouter: - """Minimal stand-in for litellm.Router.get_model_list, for policy_for_model.""" + """Minimal stand-in for litellm.Router.deployments_for_request, for policy_for_model.""" def __init__(self, deployments: list[dict[str, Any]]): self._deployments = deployments - def get_model_list(self, model_name, team_id=None): - return [d for d in self._deployments if d.get("model_name") == model_name] + def deployments_for_request(self, model, request_kwargs): + return [d for d in self._deployments if d.get("model_name") == model] def _marker(compression: dict[str, str], tags: list[str] | None = None) -> dict[str, Any]: @@ -78,23 +78,23 @@ def _marker(compression: dict[str, str], tags: list[str] | None = None) -> dict[ class TestPolicyForModel: def test_no_router_returns_none(self): - assert policy_for_model(llm_router=None, model_alias="smart-router", team_id=None, request_tags=()) is None + assert policy_for_model(llm_router=None, model_alias="smart-router", request_kwargs={}, request_tags=()) is None def test_no_marker_deployment_returns_none(self): router = _FakeRouter([{"model_name": "smart-router", "litellm_params": {"model": "openai/gpt-4o-mini"}}]) - assert policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=()) is None + assert policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=()) is None def test_marker_deployment_without_policy_returns_none(self): router = _FakeRouter( [{"model_name": "smart-router", "litellm_params": {"model": "auto_router/complexity_router"}}] ) - assert policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=()) is None + assert policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=()) is None def test_marker_deployment_with_policy_is_found(self): router = _FakeRouter( [_marker({"auto_router_routing_compression": "headroom-a", "auto_router_model_compression": "none"})] ) - policy = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=()) + policy = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=()) assert policy == AutoRouterCompressionPolicy(routing="headroom-a", model=None) def test_picks_the_marker_whose_tags_the_request_carries(self): @@ -107,8 +107,8 @@ class TestPolicyForModel: ] ) - eu = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("eu",)) - us = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("us",)) + eu = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("eu",)) + us = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("us",)) assert eu == AutoRouterCompressionPolicy(routing="headroom-eu", model=None) assert us == AutoRouterCompressionPolicy(routing="headroom-us", model=None) @@ -116,7 +116,7 @@ class TestPolicyForModel: def test_untagged_marker_matches_any_request(self): router = _FakeRouter([_marker({"auto_router_routing_compression": "headroom-a"})]) policy = policy_for_model( - llm_router=router, model_alias="smart-router", team_id=None, request_tags=("anything",) + llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("anything",) ) assert policy == AutoRouterCompressionPolicy(routing="headroom-a", model=None) @@ -128,14 +128,14 @@ class TestPolicyForModel: _marker({"auto_router_routing_compression": "headroom-default"}), ] ) - policy = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("us",)) + policy = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("us",)) assert policy == AutoRouterCompressionPolicy(routing="headroom-default", model=None) def test_no_untagged_fallback_means_no_policy(self): """No matching marker means no policy, not an unrelated slice's compression.""" router = _FakeRouter([_marker({"auto_router_routing_compression": "headroom-eu"}, tags=["eu"])]) assert ( - policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("us",)) is None + policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("us",)) is None ) def test_tag_scoped_marker_takes_precedence_over_untagged(self): @@ -147,7 +147,7 @@ class TestPolicyForModel: _marker({"auto_router_routing_compression": "headroom-eu"}, tags=["eu"]), ] ) - policy = policy_for_model(llm_router=router, model_alias="smart-router", team_id=None, request_tags=("eu",)) + policy = policy_for_model(llm_router=router, model_alias="smart-router", request_kwargs={}, request_tags=("eu",)) assert policy == AutoRouterCompressionPolicy(routing="headroom-eu", model=None) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index d2ba6527826..50b26577e5c 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -417,7 +417,7 @@ class TestProxyBaseLLMRequestProcessing: ) fake_llm_router = MagicMock() - fake_llm_router.get_model_list.return_value = [ + fake_llm_router.deployments_for_request.return_value = [ { "model_name": "smart-router", "litellm_params": { diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 9f925499f1a..51103297c58 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -3846,11 +3846,11 @@ class TestRouterPreRoutingSharedAliasName: def test_forwardable_alias_marker_params_reads_the_marker_entry_only(self): router = Router(model_list=[self._plain_entry(), self._marker_entry(), self._tier_entry()]) - forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=())) + forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=(), request_kwargs={})) assert forwarded["drop_params"] is True assert "api_key" not in forwarded and "api_base" not in forwarded - assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == () + assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=(), request_kwargs={}) == () @staticmethod def _region_marker_entry() -> dict: diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 80da922724d..1e0327e5ac2 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -10003,13 +10003,6 @@ class TestTaggedAutoRouterOnSharedModelName: def test_deployment_without_litellm_params_mapping_is_not_a_marker(self): assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False - def test_model_name_has_plain_deployments_reflects_the_pool(self): - mixed = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) - marker_only = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) - - assert mixed._model_name_has_plain_deployments("gpt4o") is True - assert marker_only._model_name_has_plain_deployments("gpt4o") is False - class TestAutoRouterSharedModelNameConnectionParams: """A plain deployment sharing its model_name with an `auto_router/` marker must not have @@ -10618,6 +10611,300 @@ class TestModelGroupAliasReachesPreRoutingStrategies: ) +class TestTeamPublicNameReachesPreRoutingStrategies: + """A team-scoped strategy router is stored under an internal `model_name_{team}_{uuid}` with the + caller-facing name in `model_info.team_public_model_name`, and the four registries key on that + internal name. A team key asks for the public name, so the hook has to resolve it to the team's + marker through the same team-first resolution the deployment path uses, and a resolution that + yields only markers is not callable on any path (LIT-7363).""" + + MARKER_TIMEOUT = 42.0 + REGISTRY_NAMES = ("auto_routers", "complexity_routers", "adaptive_routers", "quality_routers") + TEAM = "team-a" + OTHER_TEAM = "team-b" + PUBLIC_NAME = "smart-route" + INTERNAL_NAME = "model_name_team-a_0b3c" + SIBLING_INTERNAL_NAME = "model_name_team-a_9e1d" + + class _RewriteStrategy: + def __init__(self, rewrite_to: str = "gemini-flash"): + self.rewrite_to = rewrite_to + + async def async_pre_routing_hook( + self, model, request_kwargs, messages=None, input=None, specific_deployment=False + ): + from litellm.types.router import PreRoutingHookResponse + + return PreRoutingHookResponse(model=self.rewrite_to, messages=messages) + + @classmethod + def _team_marker(cls, internal_name: str, tags: list[str] | None = None) -> dict: + tiers = dict.fromkeys(("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"), "gemini-flash") + return { + "model_name": internal_name, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": tiers}, + "complexity_router_default_model": "gemini-flash", + "timeout": cls.MARKER_TIMEOUT, + **({"tags": tags} if tags else {}), + }, + "model_info": {"team_id": cls.TEAM, "team_public_model_name": cls.PUBLIC_NAME}, + } + + @classmethod + def _router( + cls, + registrations: dict[str, "TestTeamPublicNameReachesPreRoutingStrategies._RewriteStrategy"], + registry_name: str = "complexity_routers", + extra_deployments: tuple[dict, ...] = (), + markers: tuple[dict, ...] | None = None, + enable_tag_filtering: bool = False, + ) -> "litellm.Router": + from litellm.types.router import TaggedPreRoutingStrategy + + markers = markers if markers is not None else (cls._team_marker(cls.INTERNAL_NAME),) + tier = { + "model_name": "gemini-flash", + "litellm_params": {"model": "gemini/gemini-3.6-flash", "mock_response": "routed by the tier"}, + } + router = litellm.Router( + model_list=[*markers, tier, *extra_deployments], + enable_tag_filtering=enable_tag_filtering, + ) + tags_by_name = {m["model_name"]: tuple(m["litellm_params"].get("tags") or ()) for m in markers} + for name in cls.REGISTRY_NAMES: + setattr(router, name, {}) + setattr( + router, + registry_name, + { + name: [TaggedPreRoutingStrategy(tags=tags_by_name[name], strategy=strategy)] + for name, strategy in registrations.items() + }, + ) + return router + + @staticmethod + def _messages() -> list[dict[str, str]]: + return [{"role": "user", "content": "What is the capital of France?"}] + + @classmethod + def _team_request(cls, team_id: str | None = "team-a", tags: list[str] | None = None) -> dict: + metadata = {**({"user_api_key_team_id": team_id} if team_id else {}), **({"tags": tags} if tags else {})} + return {"metadata": metadata} + + @pytest.mark.parametrize("registry_name", REGISTRY_NAMES) + @pytest.mark.asyncio + async def test_team_key_dispatches_to_the_strategy_registered_under_the_internal_name(self, registry_name): + router = self._router({self.INTERNAL_NAME: self._RewriteStrategy()}, registry_name=registry_name) + + response = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(), messages=self._messages() + ) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_team_key_deployment_selection_lands_on_the_tier_and_forwards_the_marker_params(self): + router = self._router({self.INTERNAL_NAME: self._RewriteStrategy()}) + request_kwargs = self._team_request() + + deployment = await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages() + ) + + assert deployment["litellm_params"]["model"] == "gemini/gemini-3.6-flash" + assert request_kwargs["timeout"] == self.MARKER_TIMEOUT + + @pytest.mark.asyncio + async def test_another_team_never_reaches_the_strategy_or_the_marker(self): + router = self._router({self.INTERNAL_NAME: self._RewriteStrategy()}) + + assert ( + await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(self.OTHER_TEAM), messages=self._messages() + ) + is None + ) + with pytest.raises(litellm.BadRequestError): + await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(self.OTHER_TEAM), messages=self._messages() + ) + + @pytest.mark.asyncio + async def test_sibling_team_markers_select_by_request_tag_then_default(self): + router = self._router( + { + self.INTERNAL_NAME: self._RewriteStrategy("cn-model"), + self.SIBLING_INTERNAL_NAME: self._RewriteStrategy("us-model"), + }, + markers=( + self._team_marker(self.INTERNAL_NAME, tags=["cn"]), + self._team_marker(self.SIBLING_INTERNAL_NAME, tags=["us", "default"]), + ), + ) + + async def routed(tags: list[str] | None) -> str | None: + response = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(tags=tags), messages=self._messages() + ) + return response.model if response else None + + assert await routed(["cn"]) == "cn-model" + assert await routed(["us"]) == "us-model" + assert await routed(None) == "us-model" + + @pytest.mark.asyncio + async def test_team_public_name_shadows_a_global_model_for_that_team_only(self): + router = self._router( + {self.INTERNAL_NAME: self._RewriteStrategy()}, + extra_deployments=({"model_name": self.PUBLIC_NAME, "litellm_params": {"model": "openai/gpt-4o"}},), + ) + + async def routed(request_kwargs: dict) -> str | None: + response = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages() + ) + return response.model if response else None + + async def selected(request_kwargs: dict) -> str: + deployment = await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages() + ) + return deployment["litellm_params"]["model"] + + assert await routed(self._team_request()) == "gemini-flash" + assert await selected(self._team_request()) == "gemini/gemini-3.6-flash" + for request_kwargs in (self._team_request(None), self._team_request(self.OTHER_TEAM)): + assert await routed(request_kwargs) is None + assert await selected(request_kwargs) == "openai/gpt-4o" + + @pytest.mark.asyncio + async def test_tag_filtering_hands_untagged_team_requests_to_the_team_plain_sibling(self): + plain_sibling = { + "model_name": self.SIBLING_INTERNAL_NAME, + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"team_id": self.TEAM, "team_public_model_name": self.PUBLIC_NAME}, + } + router = self._router( + {self.INTERNAL_NAME: self._RewriteStrategy()}, + markers=(self._team_marker(self.INTERNAL_NAME, tags=["route"]),), + extra_deployments=(plain_sibling,), + enable_tag_filtering=True, + ) + + tagged = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(tags=["route"]), messages=self._messages() + ) + assert tagged is not None and tagged.model == "gemini-flash" + for _ in range(20): + deployment = await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(), messages=self._messages() + ) + assert deployment["litellm_params"]["model"] == "openai/gpt-4o" + + @pytest.mark.asyncio + async def test_marker_only_team_resolution_is_rejected_as_uncallable(self): + import re + + from litellm.types.router import RouterErrors + + router = self._router({}) + + with pytest.raises( + litellm.BadRequestError, match=re.escape(RouterErrors.only_strategy_marker_deployments.value) + ): + await router.async_get_available_deployment( + model=self.PUBLIC_NAME, request_kwargs=self._team_request(), messages=self._messages() + ) + + @pytest.mark.asyncio + async def test_proxy_admin_without_a_team_reaches_the_team_strategy_by_public_name(self): + router = self._router({self.INTERNAL_NAME: self._RewriteStrategy()}) + request_kwargs = {"metadata": {"user_api_key_auth": SimpleNamespace(user_role="proxy_admin")}} + + response = await router.async_pre_routing_hook( + model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages() + ) + + assert response is not None + assert response.model == "gemini-flash" + @pytest.mark.asyncio + async def test_strategy_resolution_agrees_with_the_deployment_path_for_every_principal(self): + router = self._router( + {self.INTERNAL_NAME: self._RewriteStrategy()}, + extra_deployments=({"model_name": "shared-name", "litellm_params": {"model": "openai/gpt-4o"}},), + ) + principals = { + "team": self._team_request(), + "other-team": self._team_request(self.OTHER_TEAM), + "no-team": self._team_request(None), + "admin": {"metadata": {"user_api_key_auth": SimpleNamespace(user_role="proxy_admin")}}, + } + for principal, request_kwargs in principals.items(): + for model in (self.PUBLIC_NAME, "shared-name", "gemini-flash", "missing"): + resolved = [d["model_name"] for d in router.deployments_for_request(model, request_kwargs)] + callable_names = [ + name + for name, deployment in zip(resolved, router.deployments_for_request(model, request_kwargs)) + if not router._is_strategy_marker_deployment(deployment) + ] + if resolved and not callable_names: + with pytest.raises(litellm.BadRequestError, match="strategy router marker"): + router._common_checks_available_deployment(model=model, request_kwargs=request_kwargs) + elif not resolved: + with pytest.raises(litellm.BadRequestError): + router._common_checks_available_deployment(model=model, request_kwargs=request_kwargs) + else: + _, deployments = router._common_checks_available_deployment( + model=model, request_kwargs=request_kwargs + ) + assert [d["model_name"] for d in deployments] == callable_names, (principal, model) + + def test_drop_strategy_markers_keeps_plain_deployments_and_rejects_marker_only_sets(self): + router = self._router({}) + marker = router.model_list[0] + plain = {"model_name": "plain", "litellm_params": {"model": "openai/gpt-4o"}} + + assert router._drop_strategy_markers("x", [marker, plain]) == [plain] + assert router._drop_strategy_markers("x", [plain]) == [plain] + assert router._drop_strategy_markers("x", []) == [] + with pytest.raises(litellm.BadRequestError, match="strategy router marker"): + router._drop_strategy_markers("x", [marker]) + + def test_team_deployments_across_teams_unions_one_team_and_rejects_two(self): + other_team_marker = { + **self._team_marker(self.SIBLING_INTERNAL_NAME), + "model_info": {"team_id": self.OTHER_TEAM, "team_public_model_name": self.PUBLIC_NAME}, + } + one_team = self._router({}) + two_teams = self._router({}, markers=(self._team_marker(self.INTERNAL_NAME), other_team_marker)) + + assert [d["model_name"] for d in one_team._team_deployments_across_teams(self.PUBLIC_NAME)] == [ + self.INTERNAL_NAME + ] + assert one_team._team_deployments_across_teams("missing") == [] + with pytest.raises(litellm.BadRequestError, match="multiple teams"): + two_teams._team_deployments_across_teams(self.PUBLIC_NAME) + + + def test_compression_policy_follows_the_same_resolution_for_every_principal(self): + from litellm.proxy.guardrails.auto_router_compression import AutoRouterCompressionPolicy, policy_for_model + + marker = self._team_marker(self.INTERNAL_NAME) + marker["litellm_params"]["auto_router_routing_compression"] = "headroom-team" + router = self._router({}, markers=(marker,)) + admin = {"metadata": {"user_api_key_auth": SimpleNamespace(user_role="proxy_admin")}} + expected = AutoRouterCompressionPolicy(routing="headroom-team", model=None) + + assert policy_for_model(router, self.PUBLIC_NAME, self._team_request(), ()) == expected + assert policy_for_model(router, self.PUBLIC_NAME, admin, ()) == expected + assert policy_for_model(router, self.PUBLIC_NAME, self._team_request(self.OTHER_TEAM), ()) is None + assert policy_for_model(router, self.PUBLIC_NAME, self._team_request(None), ()) is None + + class TestAutoRouterCompressionDecoupling: """An auto router's `auto_router_routing_compression` / `auto_router_model_compression` decouple what the routing decision sees from what the model call sees. The one From 07a0ca107459c0aa73da789a82e87688bf8945b9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:54:11 -0700 Subject: [PATCH 239/411] fix(e2e): bill the batch cost baton through a raw-id retrieve and drive the callback replay writer The completed marker batch is now retrieved by its raw provider id with this run's key, so the proxy prices it inline and the {provider_batch_id}_batch_cost row must join that key's token and alias. The CheckBatchCost poller only bills batches it created in the same database, which a stack booted fresh per run never holds for a completed marker, so the old unified-id assertion had no row to find. Every run also replays one callback log through POST /v1/rust_control_plane/logs, the third spend writer, and asserts its row joins the key like the eight request paths --- .../coverage_registry/quota_management.yaml | 6 +- .../spend_tracking/spend_e2e_client.py | 57 +++++- .../test_key_attribution_e2e.py | 178 +++++++++++++----- 3 files changed, 192 insertions(+), 49 deletions(-) diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 7013b59b83d..d3966eed48f 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -58,7 +58,7 @@ - {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} - {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} - {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"} -- {id: quota_management.spend_tracking.key_attribution.joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [joins_key], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Every spend row a virtual key writes across chat, queued chat, messages, responses, embeddings, the Gemini passthrough, file upload, and batch create carries api_key equal to the key's token hash and the key alias, the join the usage APIs depend on; a re-hashed token shows up as an unattributed key-hash-* row (#39568, #39572)"} -- {id: quota_management.spend_tracking.key_attribution.reports_alias_and_email, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [reports_alias_and_email], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native], source: "proxy/management_endpoints/internal_user_endpoints.py", rationale: "/spend/logs?api_key= returns every one of the key's rows with its alias and /user/daily/activity aggregates them under the key's token with key_alias and user_email; /spend/logs carries no email field, so the email is asserted on daily activity only"} +- {id: quota_management.spend_tracking.key_attribution.joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [joins_key], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Every spend row a virtual key writes across chat, queued chat, messages, responses, embeddings, the Gemini passthrough, file upload, batch create, and a replayed callback log carries api_key equal to the key's token hash and the key alias, the join the usage APIs depend on; a re-hashed token shows up as an unattributed key-hash-* row (#39568, #39572)"} +- {id: quota_management.spend_tracking.key_attribution.reports_alias_and_email, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [reports_alias_and_email], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/management_endpoints/internal_user_endpoints.py", rationale: "/spend/logs?api_key= returns every one of the key's rows with its alias and /user/daily/activity aggregates them under the key's token with key_alias and user_email; /spend/logs carries no email field, so the email is asserted on daily activity only"} - {id: quota_management.spend_tracking.key_attribution.health_rows_keep_service_account, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [health_rows_keep_service_account], exercised_on: [chat_completions], source: "proxy/health_check.py", rationale: "A /health probe's spend row stays keyed by the literal litellm-internal-health-check service account rather than a hash of it, so health spend never appears as an unattributed key"} -- {id: quota_management.spend_tracking.key_attribution.batch_cost_joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [batch_cost_joins_key], exercised_on: [batches], source: "enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py", rationale: "The _batch_cost row the batch cost poller writes once a batch completes carries the submitting key's token hash; it rides a cross-run marker batch whose metadata records the expected hash, since completion can lag by up to the polling interval. Cold start with no completed marker is a documented vacuous pass"} +- {id: quota_management.spend_tracking.key_attribution.batch_cost_joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [batch_cost_joins_key], exercised_on: [batches], source: "proxy/batches_endpoints/endpoints.py", rationale: "Retrieving a completed batch by its raw provider id prices it inline against the retrieving key, so the {provider_batch_id}_batch_cost row must carry that key's token hash and alias; the target is the newest completed cross-run marker batch this proxy has not billed yet, since completion lags by up to 24h, and a cold start with no completed marker is a documented vacuous pass. The CheckBatchCost poller's own unified-id row needs the batch create row in the same database, which a stack booted fresh per run never holds for a completed marker"} diff --git a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py index 42ebfdb98af..18b714d8aeb 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py +++ b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py @@ -17,8 +17,6 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Final -from pydantic import BaseModel, Field - from e2e_config import unique_marker from e2e_http import ( FileUploadForm, @@ -57,9 +55,12 @@ from models import ( UserRole, ) from proxy_client import Converged, ProxyClient, await_converged +from pydantic import BaseModel, Field __all__ = [ "BatchCreateBody", + "CallbackLogMetadata", + "CallbackLogPayload", "BatchObject", "DailyActivityKeyBreakdown", "FileObject", @@ -147,6 +148,44 @@ class BatchListQuery(BaseModel): limit: int +class ProviderQuery(BaseModel): + provider: str + + +class CallbackLogMetadata(BaseModel): + user_api_key_hash: str + user_api_key_alias: str + user_api_key_user_id: str + + +class CallbackLogPayload(BaseModel): + id: str + litellm_call_id: str + model: str + call_type: str = "acompletion" + start_time: float = Field(serialization_alias="startTime") + end_time: float = Field(serialization_alias="endTime") + response_cost: float + prompt_tokens: int + completion_tokens: int + total_tokens: int + metadata: CallbackLogMetadata + + +class CallbackLogRecord(BaseModel): + status: str = "success" + standard_logging_payload: CallbackLogPayload + + +class CallbackLogsRequest(BaseModel): + records: list[CallbackLogRecord] + + +class CallbackLogsResponse(BaseModel): + processed: int + failed: int + + class DailyActivityParams(BaseModel): start_date: str end_date: str @@ -445,16 +484,26 @@ class SpendClient: ) ).data - def retrieve_batch(self, key: str, batch_id: str) -> BatchObject: + def retrieve_batch(self, key: str, batch_id: str, *, provider: str) -> BatchObject: return unwrap( self.proxy.transport.get( f"/v1/batches/{batch_id}", headers=self.proxy.transport.bearer(key), - params=NoBody(), + params=ProviderQuery(provider=provider), response_type=BatchObject, ) ) + def replay_callback_log(self, key: str, payload: CallbackLogPayload) -> CallbackLogsResponse: + return unwrap( + self.proxy.transport.post( + "/v1/rust_control_plane/logs", + headers=self.proxy.transport.bearer(key), + json=CallbackLogsRequest(records=[CallbackLogRecord(standard_logging_payload=payload)]), + response_type=CallbackLogsResponse, + ) + ) + def health(self, model: str) -> ProbeResult: return self.proxy.transport.probe("/health", params=HealthParams(model=model)) diff --git a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py index 986033dbbde..cac4fd7fb3c 100644 --- a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py @@ -3,7 +3,8 @@ One virtual key with an alias, owned by a user with an email, drives every spend write path a key can reach: /chat/completions, /queue/chat/completions, /v1/messages, /v1/responses, /embeddings, the Gemini native passthrough, a batch -input file upload, and a batch create. Each row those calls write must carry +input file upload, a batch create, and a replayed callback log (POST +/v1/rust_control_plane/logs, the writer an external gateway feeds). Each row those calls write must carry `api_key` equal to the key's LiteLLM_VerificationToken.token (the sha256 hash /key/generate returns as `token`), which is the join /spend/logs?api_key= and /user/daily/activity rely on to report key_alias and user_email. A row keyed by a @@ -11,29 +12,40 @@ re-hashed token (v1.99.0's regression, #39568 and #39572) shows up as a key-hash-* row with no alias and no email in the customer's usage exports. The health-check service account writes rows too; those must stay keyed by the -literal service-account name, never by a hash of it. The batch cost row is -written by the CheckBatchCost poller once the batch completes, up to an hour -later, so it rides a cross-run baton like the batches suite: each run submits a -one-line marker batch whose metadata records the token it expects on the cost -row, and asserts on the newest completed marker from any run (a cold start with -no completed marker is a documented vacuous pass, never a skip). +literal service-account name, never by a hash of it. A batch's cost row lands +only once the batch completes, up to 24h later, so the batch cost path rides a +cross-run baton like the batches suite: each run submits a one-line marker batch +and never cancels it, and the newest completed marker from any run that this +proxy has not billed yet is retrieved by its raw provider id with this run's +key. That retrieve is the writer under test: a raw id is never poller-owned, so +the proxy prices it inline against the retrieving key, and the +{provider_batch_id}_batch_cost row must join this run's token with its alias. +The CheckBatchCost poller's own row (the unified id, billed against the +submitting key) needs the batch's create row in the same database, which a stack +booted fresh per run never holds for a completed marker, so that writer is out +of this test's reach. A cold start with no completed marker is a documented +vacuous pass, never a skip. /spend/logs carries no email field, so the email assertion lives on /user/daily/activity alone; /spend/logs is held to the alias in metadata. """ +import base64 import time +from collections.abc import Iterator from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import Final, Iterator +from typing import Final import pytest - -from models import ChatMessage, KeyGenerateBody +from models import ChatMessage, KeyGenerateBody, SpendLogsParams from proxy_client import Converged, await_converged +from pydantic import BaseModel from spend_e2e_client import ( BatchCreateBody, BatchObject, + CallbackLogMetadata, + CallbackLogPayload, DailyActivityKeyBreakdown, ResponseIdentity, SpendClient, @@ -41,7 +53,6 @@ from spend_e2e_client import ( StreamingResponse, unique_marker, ) -from pydantic import BaseModel pytestmark = pytest.mark.e2e @@ -51,14 +62,17 @@ RESPONSES_MODEL: Final = "openai-responses-codex" EMBED_MODEL: Final = "openai-text-embedding-3-small" BATCH_MODEL: Final = "openai-gpt-4o-mini" BATCH_BACKEND_MODEL: Final = "gpt-4o-mini" +BATCH_PROVIDER: Final = "openai" HEALTH_SERVICE_ACCOUNT: Final = "litellm-internal-health-check" BATON_MARKER_KEY: Final = "litellm_e2e_suite" BATON_MARKER_VALUE: Final = "key-attribution-baton" -BATON_EXPECTED_TOKEN_KEY: Final = "expected_api_key" -BATON_POLL_SECONDS: Final = 300.0 +BATON_POLL_SECONDS: Final = 30.0 BATON_POLL_INTERVAL_SECONDS: Final = 10.0 BATON_LIST_LIMIT: Final = 100 MAX_TOKENS: Final = 8 +REPLAY_RESPONSE_COST: Final = 0.0001 +REPLAY_PROMPT_TOKENS: Final = 5 +REPLAY_COMPLETION_TOKENS: Final = 1 WRITE_PATHS: Final = ( "chat_completions", "queue_chat_completions", @@ -68,6 +82,7 @@ WRITE_PATHS: Final = ( "gemini_passthrough", "batch_file_upload", "batch_create", + "callback_replay", ) @@ -138,11 +153,7 @@ def _drive_batch(client: SpendClient, identity: AttributedKey, marker: str) -> t BatchCreateBody( input_file_id=uploaded.id, model=BATCH_MODEL, - metadata={ - BATON_MARKER_KEY: BATON_MARKER_VALUE, - BATON_EXPECTED_TOKEN_KEY: identity.token, - "run": marker, - }, + metadata={BATON_MARKER_KEY: BATON_MARKER_VALUE, "run": marker}, ), ) return ( @@ -151,6 +162,32 @@ def _drive_batch(client: SpendClient, identity: AttributedKey, marker: str) -> t ) +def _drive_callback_replay(client: SpendClient, identity: AttributedKey, marker: str) -> WritePath: + request_id: Final = f"callback-replay-{marker}" + finished_at: Final = time.time() + replayed: Final = client.replay_callback_log( + identity.key, + CallbackLogPayload( + id=request_id, + litellm_call_id=request_id, + model=CHAT_MODEL, + start_time=finished_at - 1, + end_time=finished_at, + response_cost=REPLAY_RESPONSE_COST, + prompt_tokens=REPLAY_PROMPT_TOKENS, + completion_tokens=REPLAY_COMPLETION_TOKENS, + total_tokens=REPLAY_PROMPT_TOKENS + REPLAY_COMPLETION_TOKENS, + metadata=CallbackLogMetadata( + user_api_key_hash=identity.token, + user_api_key_alias=identity.alias, + user_api_key_user_id=identity.user_id, + ), + ), + ) + assert replayed.processed == 1 and replayed.failed == 0, f"callback replay rejected the payload: {replayed}" + return WritePath(name="callback_replay", request_id=request_id) + + def _drive_every_write_path(client: SpendClient, identity: AttributedKey) -> tuple[WritePath, ...]: marker: Final = unique_marker() prompt: Final = f"Reply with the word ok. {marker}" @@ -163,31 +200,52 @@ def _drive_every_write_path(client: SpendClient, identity: AttributedKey) -> tup _call_id("embeddings", client.send_embed(key, EMBED_MODEL, prompt)), _call_id("gemini_passthrough", client.send_gemini_generate(key, CHAT_MODEL, prompt, max_tokens=MAX_TOKENS)), *_drive_batch(client, identity, marker), + _drive_callback_replay(client, identity, marker), ) -def _completed_baton(listed: list[BatchObject]) -> BatchObject | None: - return max( - ( - batch - for batch in listed - if batch.status == "completed" and (batch.metadata or {}).get(BATON_MARKER_KEY) == BATON_MARKER_VALUE - ), - key=lambda batch: batch.created_at or 0, - default=None, +def _completed_batons(listed: list[BatchObject]) -> tuple[BatchObject, ...]: + return tuple( + sorted( + ( + batch + for batch in listed + if batch.status == "completed" and (batch.metadata or {}).get(BATON_MARKER_KEY) == BATON_MARKER_VALUE + ), + key=lambda batch: batch.created_at or 0, + reverse=True, + ) ) -def _newest_completed_baton(client: SpendClient, key: str) -> BatchObject | None: +def _provider_batch_id(unified_batch_id: str) -> str: + encoded: Final = unified_batch_id.removeprefix("batch_") + decoded: Final = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)).decode() + return decoded.removeprefix("litellm:").split(";", 1)[0] + + +def _batch_cost_request_id(unified_batch_id: str) -> str: + return f"{_provider_batch_id(unified_batch_id)}_batch_cost" + + +def _newest_unbilled_completed_baton(client: SpendClient, key: str) -> BatchObject | None: outcome: Final = await_converged( - lambda: _completed_baton(client.list_batches(key, BATCH_MODEL, limit=BATON_LIST_LIMIT)), - converged=lambda completed: completed is not None, + lambda: _completed_batons(client.list_batches(key, BATCH_MODEL, limit=BATON_LIST_LIMIT)), + converged=lambda completed: bool(completed), timeout=BATON_POLL_SECONDS, interval=BATON_POLL_INTERVAL_SECONDS, now=time.monotonic, sleep=time.sleep, ) - return outcome.result if isinstance(outcome, Converged) else None + completed: Final = outcome.result if isinstance(outcome, Converged) else () + return next( + ( + batch + for batch in completed + if not client.proxy.spend_logs(SpendLogsParams(request_id=_batch_cost_request_id(batch.id))) + ), + None, + ) def _health_rows_between(client: SpendClient, started_at: datetime) -> list[SpendLogRow]: @@ -246,7 +304,16 @@ class TestKeyAttribution: @pytest.mark.covers( "quota_management.spend_tracking.key_attribution.joins_key", - exercised_on=["chat_completions", "messages", "responses", "embeddings", "batches", "files", "google_native"], + exercised_on=[ + "chat_completions", + "messages", + "responses", + "embeddings", + "batches", + "files", + "google_native", + "rust_control_plane", + ], ) def test_every_write_path_row_joins_the_key(self, client: SpendClient, driven: DrivenKey) -> None: assert tuple(path.name for path in driven.paths) == WRITE_PATHS @@ -275,7 +342,16 @@ class TestKeyAttribution: @pytest.mark.covers( "quota_management.spend_tracking.key_attribution.reports_alias_and_email", - exercised_on=["chat_completions", "messages", "responses", "embeddings", "batches", "files", "google_native"], + exercised_on=[ + "chat_completions", + "messages", + "responses", + "embeddings", + "batches", + "files", + "google_native", + "rust_control_plane", + ], ) def test_spend_logs_by_key_return_every_row_with_the_alias(self, client: SpendClient, driven: DrivenKey) -> None: expected_ids: Final = frozenset(path.request_id for path in driven.paths) @@ -294,7 +370,16 @@ class TestKeyAttribution: @pytest.mark.covers( "quota_management.spend_tracking.key_attribution.reports_alias_and_email", - exercised_on=["chat_completions", "messages", "responses", "embeddings", "batches", "files", "google_native"], + exercised_on=[ + "chat_completions", + "messages", + "responses", + "embeddings", + "batches", + "files", + "google_native", + "rust_control_plane", + ], ) def test_user_daily_activity_reports_alias_and_email(self, client: SpendClient, driven: DrivenKey) -> None: breakdown: Final[DailyActivityKeyBreakdown | None] = client.poll_daily_activity_for_key( @@ -332,18 +417,27 @@ class TestKeyAttribution: exercised_on=["batches"], ) def test_completed_batch_cost_row_joins_the_key(self, client: SpendClient, driven: DrivenKey) -> None: - completed: Final = _newest_completed_baton(client, driven.identity.key) + completed: Final = _newest_unbilled_completed_baton(client, driven.identity.key) if completed is None: return - expected_token: Final = (completed.metadata or {}).get(BATON_EXPECTED_TOKEN_KEY) - assert expected_token, f"marker batch {completed.id} lost its expected token metadata: {completed.metadata}" - fetched: Final = client.retrieve_batch(driven.identity.key, completed.id) + provider_batch_id: Final = _provider_batch_id(completed.id) + fetched: Final = client.retrieve_batch(driven.identity.key, provider_batch_id, provider=BATCH_PROVIDER) assert fetched.status == "completed", f"listed-completed marker retrieved as {fetched.status!r}" + cost_request_id: Final = _batch_cost_request_id(completed.id) rows: Final = client.proxy.poll_logs_for_request_id( - f"{fetched.id}_batch_cost", + cost_request_id, predicate=lambda found: any((row.spend or 0) > 0 for row in found), ) priced: Final = [row for row in rows if (row.spend or 0) > 0] - assert priced, f"completed batch {fetched.id} has no positive-cost spend row under {fetched.id}_batch_cost" - unjoined: Final = [(row.call_type, row.api_key) for row in priced if row.api_key != expected_token] - assert not unjoined, f"batch cost rows whose api_key does not join the key's token {expected_token}: {unjoined}" + assert priced, f"retrieving completed batch {provider_batch_id} wrote no positive-cost row under {cost_request_id}" + unjoined: Final = [ + (row.call_type, row.api_key, row.metadata.user_api_key_alias if row.metadata else None) + for row in priced + if row.api_key != driven.identity.token + or row.metadata is None + or row.metadata.user_api_key_alias != driven.identity.alias + ] + assert not unjoined, ( + f"batch cost rows that do not join the retrieving key's token {driven.identity.token} " + f"with alias {driven.identity.alias!r}: {unjoined}" + ) From fc161faa96a27f16d90d1aee6e4fc0574f679dd9 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 9 Sep 2026 15:56:38 -0700 Subject: [PATCH 240/411] fix(langfuse): give each call in a session header its own trace instead of upserting one trace per session (#40177) * fix(langfuse): give session-header calls their own trace id A client that sends only a session header (x-litellm-session-id, a vendor x--session-id such as Claude Code's X-Claude-Code-Session-Id, a bare x-session-id, or the Codex session/thread/conversation family) has that value stamped into both trace_id and session_id by the proxy. Langfuse upserts a trace by id, so every turn of a session collapsed into one growing trace and the Sessions view showed "Total traces: 1" Detect that aliasing in the Langfuse callback from the request headers the callback already receives, and use litellm_call_id as the trace id for those calls. session_id still carries the header value, so the turns stay grouped under one session. An explicit x-litellm-trace-id, langfuse_trace_id, or langfuse_existing_trace_id keeps its trace id, including when the caller sets it to the same value as the session id Co-authored-by: jesus * test(langfuse): cover direct-SDK callers without proxy request headers * fix(langfuse): preserve session trace provenance --------- Co-authored-by: jesus --- litellm/integrations/langfuse/langfuse.py | 59 +++- .../integrations/test_langfuse.py | 251 ++++++++++++++++++ 2 files changed, 306 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 9576eabaa34..b75369965de 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -2,6 +2,7 @@ # On success, logs events to Langfuse import inspect import os +import re import traceback from collections.abc import Callable, Iterable, Mapping from datetime import datetime @@ -63,6 +64,44 @@ def _object_mapping(value: object) -> Mapping[str, object] | None: return value if isinstance(value, dict) else None +def _widened_items(mapping: Mapping[str, object]) -> Iterable[tuple[object, object]]: + """Header pairs with the key type widened back to what a caller-supplied dict can actually hold.""" + return mapping.items() + + +def _is_session_header_trace(trace_id: object, session_id: object, proxy_server_request: object) -> bool: + if not isinstance(trace_id, str) or not isinstance(session_id, str): + return False + request: Final = _object_mapping(proxy_server_request) + raw_headers: Final = _object_mapping(request.get("headers")) if request is not None else None + if raw_headers is None: + return False + headers: Final = MappingProxyType( + {key.lower(): value for key, value in _widened_items(raw_headers) if isinstance(key, str)} + ) + if headers.get("x-litellm-trace-id"): + return False + if headers.get("langfuse_trace_id") is not None: + return False + if trace_id != session_id and headers.get("langfuse_session_id") != session_id: + return False + if headers.get("x-litellm-session-id") == trace_id: + return True + if re.fullmatch(r"[a-zA-Z0-9_\-]{8,}", trace_id) is None: + return False + user_agent: Final = headers.get("user-agent") + codex: Final = isinstance(user_agent, str) and re.match(r"^codex[-_ /]", user_agent, re.IGNORECASE) is not None + return any( + value == trace_id + and ( + key == "x-session-id" + or re.fullmatch(r"x-.+-session-id", key) is not None + or (codex and key in ("session-id", "session_id", "thread-id", "conversation_id")) + ) + for key, value in headers.items() + ) + + class _UsageObject(Protocol): """Token-count surface the Langfuse logger reads off a response usage payload.""" @@ -609,6 +648,18 @@ class LangFuseLogger: # This allows continuing an existing trace while still returning the correct trace_id if existing_trace_id is not None: trace_id = existing_trace_id + resolved_trace_id: Final = ( + litellm_call_id or trace_id + if existing_trace_id is None + and _is_session_header_trace(trace_id, session_id, litellm_params.get("proxy_server_request")) + else trace_id + ) + if resolved_trace_id != trace_id: + verbose_logger.debug( + "Langfuse: trace_id %s came from a session header; using call id %s so each call gets its own trace", + trace_id, + resolved_trace_id, + ) requested_trace_keys: Final = _as_steering_key_sequence(clean_metadata.pop("update_trace_keys", ())) update_trace_keys: Final = ( requested_trace_keys if _as_steering_flag(litellm.langfuse_enable_update_trace_keys) else () @@ -663,7 +714,7 @@ class LangFuseLogger: trace_params["output"] = masked_output if not mask_output else "redacted-by-litellm" else: # don't overwrite an existing trace trace_params = { - "id": trace_id, + "id": resolved_trace_id, "name": trace_name, "session_id": session_id, "input": masked_input if not mask_input else "redacted-by-litellm", @@ -845,13 +896,13 @@ class LangFuseLogger: # Verify langfuse accepted our trace_id; if it differs, log a warning but still return our intended value # to match expected test behavior if hasattr(generation_client, "trace_id") and generation_client.trace_id: - if generation_client.trace_id != trace_id: + if generation_client.trace_id != resolved_trace_id: verbose_logger.warning( "Langfuse trace_id mismatch: set %s, but langfuse returned %s. Using our intended trace_id for consistency.", - trace_id, + resolved_trace_id, generation_client.trace_id, ) - return trace_id, generation_id + return resolved_trace_id, generation_id except Exception: verbose_logger.error("Langfuse Layer Error - %s", traceback.format_exc()) return None, None diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index d36878e455f..87e76499b84 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -1341,6 +1341,257 @@ def _emit(logger: LangFuseLogger, *, metadata=None, headers=None): ) +@pytest.mark.parametrize("level", ["DEFAULT", "ERROR"]) +@pytest.mark.parametrize( + "headers,metadata,expected_id", + [ + ({"x-litellm-session-id": "session-7125"}, {}, "call"), + ({"X-Claude-Code-Session-Id": "session-7125"}, {}, "call"), + ({"x-session-id": "session-7125"}, {}, "call"), + ({"session-id": "session-7125", "user-agent": "codex_cli_rs/1.0"}, {}, "call"), + ({"thread-id": "session-7125", "user-agent": "codex-tui"}, {}, "call"), + ({"session_id": "session-7125", "user-agent": "Codex 1.0"}, {}, "call"), + ({"conversation_id": "session-7125", "user-agent": "codex_vscode/1.0"}, {}, "call"), + ({"x-litellm-session-id": "short"}, {}, "call"), + ({"x-litellm-trace-id": "session-7125"}, {}, "session-7125"), + ( + {"X-LiteLLM-Trace-Id": "session-7125", "x-litellm-session-id": "session-7125"}, + {}, + "session-7125", + ), + ( + {"x-litellm-session-id": "session-7125", "langfuse_trace_id": "session-7125"}, + {}, + "session-7125", + ), + ( + {"x-litellm-session-id": "session-7125", "langfuse_trace_id": "explicit-trace"}, + {}, + "explicit-trace", + ), + ( + {"x-litellm-session-id": "session-7125", "langfuse_existing_trace_id": "existing-trace"}, + {}, + "existing-trace", + ), + ( + {"x-litellm-session-id": "session-7125", "langfuse_session_id": "custom-session"}, + {}, + "call", + ), + ( + {"x-litellm-session-id": "short", "langfuse_session_id": "custom-session"}, + {}, + "call", + ), + ( + {"X-Claude-Code-Session-Id": "session-7125", "langfuse_session_id": "custom-session"}, + {}, + "call", + ), + ( + {"x-session-id": "session-7125", "langfuse_session_id": "custom-session"}, + {}, + "call", + ), + ( + { + "session-id": "session-7125", + "user-agent": "codex_cli_rs/1.0", + "langfuse_session_id": "custom-session", + }, + {}, + "call", + ), + ( + { + "x-litellm-session-id": "session-7125", + "langfuse_session_id": "custom-session", + "x-litellm-trace-id": "explicit-trace", + }, + {}, + "explicit-trace", + ), + ( + { + "x-litellm-session-id": "session-7125", + "langfuse_session_id": "custom-session", + "langfuse_trace_id": "explicit-trace", + }, + {}, + "explicit-trace", + ), + ( + { + "x-litellm-session-id": "session-7125", + "langfuse_session_id": "custom-session", + "langfuse_existing_trace_id": "existing-trace", + }, + {}, + "existing-trace", + ), + ({}, {"trace_id": "session-7125", "session_id": "session-7125"}, "session-7125"), + ({}, {"trace_id": "explicit-trace", "session_id": "session-7125"}, "explicit-trace"), + ( + {"x-vendor-session-id": "short"}, + {"trace_id": "short", "session_id": "short"}, + "short", + ), + ( + {"x-session-id": "invalid value"}, + {"trace_id": "invalid value", "session_id": "invalid value"}, + "invalid value", + ), + ( + {"session-id": "session-7125", "user-agent": "codexfoo/1.0"}, + {"trace_id": "session-7125", "session_id": "session-7125"}, + "session-7125", + ), + ( + {"x-vendor-session-id": "short"}, + {"trace_id": "session-7125", "session_id": "session-7125"}, + "session-7125", + ), + ({}, {}, "call"), + ], +) +def test_session_header_trace_provenance(headers, metadata, expected_id, level): + from starlette.datastructures import Headers + + from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + clean_headers, + redact_credential_headers, + ) + + logger: Final = _steering_logger() + for turn in range(2): + call_id = f"call-{turn}" + request_headers = Headers(headers) + data = LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=request_headers, data={"metadata": dict(metadata)}, _metadata_variable_name="metadata" + ) + original_metadata = dict(data["metadata"]) + now = datetime.datetime.now() + result = logger.log_event_on_langfuse( + kwargs={ + "call_type": "completion", + "litellm_call_id": call_id, + "litellm_trace_id": data.get("litellm_trace_id"), + "litellm_params": { + "metadata": data["metadata"], + "proxy_server_request": {"headers": redact_credential_headers(clean_headers(request_headers))}, + }, + "messages": [{"role": "user", "content": f"turn {turn}"}], + "optional_params": {}, + }, + response_obj=( + None + if level == "ERROR" + else litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "OK"}}]) + ), + start_time=now, + end_time=now, + level=level, + status_message="provider error" if level == "ERROR" else None, + ) + trace_params = logger.Langfuse.trace.call_args.kwargs + assert trace_params["id"] == (call_id if expected_id == "call" else expected_id) + assert result["trace_id"] == trace_params["id"] + if expected_id != "existing-trace": + assert trace_params["session_id"] == headers.get("langfuse_session_id", original_metadata.get("session_id")) + steering = {key[len("langfuse_") :]: value for key, value in headers.items() if key.startswith("langfuse_")} + assert data["metadata"] == {**original_metadata, **steering} + + +def test_session_header_trace_without_call_id_keeps_session_alias(): + logger: Final = _steering_logger() + now: Final = datetime.datetime.now() + + result: Final = logger.log_event_on_langfuse( + kwargs={ + "call_type": "completion", + "litellm_call_id": "", + "litellm_params": { + "metadata": {"trace_id": "session-7125", "session_id": "session-7125"}, + "proxy_server_request": {"headers": {"x-litellm-session-id": "session-7125"}}, + }, + "messages": [{"role": "user", "content": "no call id"}], + "optional_params": {}, + }, + response_obj=litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "OK"}}]), + start_time=now, + end_time=now, + ) + + assert logger.Langfuse.trace.call_args.kwargs["id"] == "session-7125" + assert result["trace_id"] == "session-7125" + + +def test_every_proxy_session_header_shape_is_classified_as_a_session_alias(): + """The classifier must cover every header shape the proxy turns into a chain id.""" + from litellm.integrations.langfuse.langfuse import _is_session_header_trace + from litellm.proxy.litellm_pre_call_utils import ( + _CODEX_SESSION_ID_HEADERS, + get_chain_id_from_headers, + ) + + session: Final = "session-7125-abcdef" + session_shapes: Final = ( + {"x-litellm-session-id": session}, + {"X-Claude-Code-Session-Id": session}, + {"x-session-id": session}, + *({header: session, "user-agent": "codex_cli_rs/1.0"} for header in _CODEX_SESSION_ID_HEADERS), + ) + for headers in session_shapes: + assert get_chain_id_from_headers(dict(headers)) == session, headers + assert _is_session_header_trace(session, session, {"headers": headers}) is True, headers + + explicit_trace: Final = {"x-litellm-trace-id": session, "x-litellm-session-id": session} + assert get_chain_id_from_headers(dict(explicit_trace)) == session + assert _is_session_header_trace(session, session, {"headers": explicit_trace}) is False + + +@pytest.mark.parametrize( + "proxy_server_request", + [None, {}, {"headers": None}], + ids=["no-proxy-request", "no-headers-key", "null-headers"], +) +def test_sdk_caller_without_request_headers_keeps_its_trace(proxy_server_request): + """A direct SDK caller has no request headers, so a session-shaped trace id stays the caller's.""" + logger: Final = _steering_logger() + now: Final = datetime.datetime.now() + + result: Final = logger.log_event_on_langfuse( + kwargs={ + "call_type": "completion", + "litellm_call_id": "call-0", + "litellm_params": { + "metadata": {"trace_id": "session-7125", "session_id": "session-7125"}, + "proxy_server_request": proxy_server_request, + }, + "messages": [{"role": "user", "content": "sdk turn"}], + "optional_params": {}, + }, + response_obj=litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "OK"}}]), + start_time=now, + end_time=now, + ) + + assert logger.Langfuse.trace.call_args.kwargs["id"] == "session-7125" + assert result["trace_id"] == "session-7125" + + +def test_session_header_classifier_survives_non_string_header_keys(): + """A non-string header key must not cost the caller its whole trace.""" + from litellm.integrations.langfuse.langfuse import _is_session_header_trace + + session: Final = "session-7125-abcdef" + headers: Final = {7: "numeric key", "x-litellm-session-id": session} + assert _is_session_header_trace(session, session, {"headers": headers}) is True + assert _is_session_header_trace(session, session, {"headers": {7: "numeric key"}}) is False + + def test_mask_input_header_false_keeps_the_prompt(): logger = _steering_logger() From 21e6c6e2f3c9fe993925b6aaaca6314bd56b1add Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:56:41 -0700 Subject: [PATCH 241/411] test(convert_dict_to_response): expect the type-naming error for a null choices value --- .../test_convert_dict_to_chat_completion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index 09d1adba17c..31c554985a7 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -1645,7 +1645,7 @@ class TestMissingChoicesGuard: assert result.usage.prompt_tokens == 10 def test_convert_to_model_response_object_null_choices_raises_api_error(self): - """choices=None raises APIError.""" + """choices=None raises APIError that names the type instead of claiming the key is missing.""" from litellm.exceptions import APIError response_object = { @@ -1661,7 +1661,7 @@ class TestMissingChoicesGuard: model_response_object=ModelResponse(), ) - assert "no 'choices'" in exc_info.value.message + assert "'choices' that is not a list (NoneType)" in exc_info.value.message def test_convert_to_streaming_response_no_choices_raises_api_error(self): """Missing choices in streaming cache-hit path raises APIError.""" From fbc6fb56ae2d8d8e600b0e80f7a46067b796d47f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:06:19 -0700 Subject: [PATCH 242/411] fix(bedrock): route gpt-6-astra reasoning_effort to reasoning.effort and mark Nova 2 tool_choice The converse reasoning gate only matched openai.gpt-5, so gpt-6-astra fell through to Anthropic's thinking block and Bedrock rejected the call with 400 Unknown parameter: 'thinking'. Match any openai.gpt- model at the three gate sites instead. Nova 2 lite and pro accept forced tool_choice on Converse (verified live on us.amazon.nova-2-lite-v1:0), so the nine Nova 2 registry keys now advertise supports_tool_choice. The invoke dispatcher also forwards json_mode to Nova like it already does for Anthropic and TwelveLabs. --- .../bedrock/chat/converse_transformation.py | 17 +++++-- .../base_invoke_transformation.py | 1 + ...odel_prices_and_context_window_backup.json | 9 ++++ model_prices_and_context_window.json | 9 ++++ .../test_base_invoke_transformation.py | 44 +++++++++++++++++++ .../chat/test_converse_transformation.py | 3 ++ 6 files changed, 79 insertions(+), 4 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6173f08fc87..4cdf387a11d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -4,6 +4,7 @@ Translating between OpenAI's `/chat/completion` format and Amazon's `/converse` import copy import json +import re import time import types from collections.abc import Mapping @@ -293,6 +294,10 @@ class AmazonConverseConfig(BaseConfig): llm_provider="bedrock", ) + @staticmethod + def _is_openai_gpt_reasoning_model(model: str) -> bool: + return re.search(r"openai\.gpt-\d", model) is not None + def _is_nova_2_model(self, model: str) -> bool: """ Check if the model is a Nova 2 model that supports reasoningConfig. @@ -423,14 +428,14 @@ class AmazonConverseConfig(BaseConfig): Handle the reasoning_effort parameter based on the model type. - GPT-OSS models: passed through unchanged via additionalModelRequestFields. - - OpenAI GPT-5.x models: mapped to ``reasoning.effort`` via additionalModelRequestFields. + - OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields. - Nova 2 models: transformed to reasoningConfig. - Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on adaptive Claude 4.6 / 4.7). """ if "gpt-oss" in model: optional_params["reasoning_effort"] = reasoning_effort - elif "openai.gpt-5" in model: + elif self._is_openai_gpt_reasoning_model(model): reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort} optional_params["reasoning"] = reasoning elif self._is_nova_2_model(model): @@ -564,7 +569,11 @@ class AmazonConverseConfig(BaseConfig): # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html supported_params.append("tool_choice") - if "gpt-oss" in model or "openai.gpt-5" in model or "openai.gpt-5" in base_model: + if ( + "gpt-oss" in model + or self._is_openai_gpt_reasoning_model(model) + or self._is_openai_gpt_reasoning_model(base_model) + ): supported_params.append("reasoning_effort") elif self._is_nova_2_model(model): # Nova 2 models support reasoning_effort (transformed to reasoningConfig) @@ -920,7 +929,7 @@ class AmazonConverseConfig(BaseConfig): optional_params["_parallel_tool_use_config"] = { "tool_choice": {"type": "auto", "disable_parallel_tool_use": not value} } - if param == "thinking" and "openai.gpt-5" not in model: + if param == "thinking" and not self._is_openai_gpt_reasoning_model(model): if ( isinstance(value, dict) and value.get("type") == "adaptive" diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index a0e32c8aa22..90a2692f68a 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -340,6 +340,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): optional_params=optional_params, litellm_params=litellm_params, encoding=encoding, + json_mode=json_mode, ) elif provider == "twelvelabs": return litellm.AmazonTwelveLabsPegasusConfig().transform_response( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a2e7b692649..0d2eda93323 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -381,6 +381,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -400,6 +401,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -417,6 +419,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -436,6 +439,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -453,6 +457,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -472,6 +477,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -489,6 +495,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -508,6 +515,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -28578,6 +28586,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a2e7b692649..0d2eda93323 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -381,6 +381,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -400,6 +401,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -417,6 +419,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -436,6 +439,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -453,6 +457,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -472,6 +477,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -489,6 +495,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -508,6 +515,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, @@ -28578,6 +28586,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index c2c448cd7e2..96a2fa6ec67 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -1,5 +1,7 @@ import json +from unittest.mock import MagicMock +import httpx import pytest @@ -190,3 +192,45 @@ def test_get_error_class_preserves_provider_headers(): assert isinstance(error, BedrockError) assert error.headers == {"x-amzn-RequestId": "req-invoke-500"} assert error.response.headers["x-amzn-requestid"] == "req-invoke-500" + + +def test_transform_response_hands_json_mode_to_nova(): + """The invoke dispatcher forwards its json_mode argument to Nova instead of dropping it.""" + from litellm.types.utils import ModelResponse + + response_json = { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_nova_json", + "name": "json_tool_call", + "input": {"city": "Paris", "temperature": 21}, + } + } + ], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 5, "outputTokens": 4, "totalTokens": 9}, + } + raw_response = httpx.Response(200, json=response_json, request=httpx.Request("POST", "https://bedrock")) + + result = AmazonInvokeConfig().transform_response( + model="invoke/amazon.nova-lite-v1:0", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "weather"}], + optional_params={}, + litellm_params={}, + encoding=None, + api_key=None, + json_mode=True, + ) + + assert result.choices[0].message.tool_calls is None + assert json.loads(result.choices[0].message.content) == {"city": "Paris", "temperature": 21} diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 16824d19f23..51c0cd261e8 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -382,6 +382,8 @@ def test_reasoning_with_forced_tool_choice_switches_to_auto(): "us.openai.gpt-5.6-sol", "global.openai.gpt-5.6-terra", "bedrock/converse/us.openai.gpt-5.6-luna", + "us.openai.gpt-6-astra", + "bedrock/converse/global.openai.gpt-6-astra", ], ) def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(model, local_model_cost_map): @@ -412,6 +414,7 @@ def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(mode [ "us.openai.gpt-5.6-sol", "bedrock/converse/global.openai.gpt-5.6-luna", + "us.openai.gpt-6-astra", ], ) def test_openai_gpt5_converse_never_forwards_thinking(model, local_model_cost_map): From 96e46ba8ff43cdbf24d53691685c15b533bfa746 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:06:55 -0700 Subject: [PATCH 243/411] fix(responses): stamp streamed usage cost when the provider usage arrives as a dict Perplexity's Responses payload fails ResponsesAPIResponse validation on truncation "" and is kept as an unvalidated model, so its usage stays a plain dict and _stamp_responses_usage_cost raised AttributeError on every streamed completion once reasoning made the cost non-zero. Validate the dict into ResponseAPIUsage before stamping, keeping a provider-reported cost when it carries one. Resolves LIT-7391 --- litellm/responses/streaming_iterator.py | 17 ++++++- .../responses/test_streaming_iterator.py | 51 +++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 9f9016c5a7f..658a0e8ad5d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload, runti import httpx from openai._streaming import SSEDecoder +from pydantic import ValidationError from typing_extensions import TypeIs import litellm @@ -544,7 +545,7 @@ class BaseResponsesAPIStreamingIterator: def _record_failed_response_usage(self, response_obj: ResponsesAPIResponse | None) -> None: if response_obj is None or self.logging_obj is None: return - usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) + usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None)) if usage_obj is None: return try: @@ -1293,14 +1294,26 @@ def _add_text_like_part_events( ) +def _usage_as_model(usage: object) -> ResponseAPIUsage | None: + if isinstance(usage, ResponseAPIUsage): + return usage + if not isinstance(usage, dict): + return None + try: + return ResponseAPIUsage.model_validate(usage) + except ValidationError: + return None + + def _stamp_responses_usage_cost( response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None ) -> None: if response_obj is None or logging_obj is None: return - usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) + usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None)) if usage_obj is None: return + response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives if isinstance(getattr(usage_obj, "cost", None), (int, float)): return try: diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index c226c0b4d09..60ce08666b6 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -368,6 +368,57 @@ def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): logging_obj._response_cost_calculator.assert_not_called() +def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: + return ResponsesAPIResponse.model_construct( + id="resp_lit7391", + created_at=int(datetime(2025, 1, 1).timestamp()), + status="completed", + model="perplexity/deepseek-v4-flash-0731", + object="response", + output=[], + truncation="", + usage=usage, + ) + + +def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + from litellm.types.llms.openai import ResponseAPIUsage + + response = _unvalidated_response_with_dict_usage( + { + "input_tokens": 29, + "output_tokens": 120, + "output_tokens_details": {"reasoning_tokens": 117}, + "total_tokens": 149, + "cost": {"currency": "USD", "input_cost": 0, "output_cost": 3e-05, "total_cost": 3e-05}, + } + ) + logging_obj = Mock(spec=LiteLLMLoggingObj) + + _stamp_responses_usage_cost(response, logging_obj) + + assert isinstance(response.usage, ResponseAPIUsage) + assert response.usage.cost == pytest.approx(3e-05) + assert response.usage.output_tokens_details.reasoning_tokens == 117 + logging_obj._response_cost_calculator.assert_not_called() + + +def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + from litellm.types.llms.openai import ResponseAPIUsage + + response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149}) + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj._response_cost_calculator.return_value = 0.000704 + + _stamp_responses_usage_cost(response, logging_obj) + + assert isinstance(response.usage, ResponseAPIUsage) + assert response.usage.cost == pytest.approx(0.000704) + logging_obj._response_cost_calculator.assert_called_once_with(result=response) + + def test_stamp_responses_usage_cost_survives_calculator_failure(): from litellm.responses.streaming_iterator import _stamp_responses_usage_cost From b1fb478e37c1a06fef77ccabdfdf3abb7666c29a Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 9 Sep 2026 16:10:04 -0700 Subject: [PATCH 244/411] feat(proxy): restrict aws_session_tags on model management to proxy admins Team admins can create and edit team models through /model/new, PUT /model/update and PATCH /model/{id}/update. The proxy forwards aws_session_tags to STS under its own identity, so a team admin could pick tags that unlock aws:PrincipalTag gated resources. Only proxy admins may now set or change aws_session_tags there; an unchanged tag set still passes so team admins can edit other fields --- .../model_management_endpoints.py | 44 +++++ .../test_model_management_endpoints.py | 179 ++++++++++++++++++ 2 files changed, 223 insertions(+) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 0f19e9ce149..94d2b773e14 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -111,6 +111,7 @@ from litellm.router_utils.auto_router_model_naming import ( validate_strategy_router_model_write, ) from litellm.router_utils.auto_router_tuning_baseline import is_mutable_tuned_candidate, tuning_quota_violation +from litellm.types.llms.bedrock import AwsSessionTag from litellm.types.proxy.management_endpoints.model_management_endpoints import ( AutoRouterClassifierDefaultPromptResponse, UpdateUsefulLinksRequest, @@ -897,6 +898,12 @@ async def patch_model( existing_litellm_params=db_model.litellm_params, ) + ModelManagementAuthChecks.can_user_set_aws_session_tags( + litellm_params=patch_data.litellm_params, + user_api_key_dict=user_api_key_dict, + existing_litellm_params=db_model.litellm_params, + ) + _raise_on_strategy_router_write_violation( incoming_params=patch_data.litellm_params, existing_params=db_model.litellm_params, @@ -1650,6 +1657,10 @@ async def _update_existing_team_model_assignment( # No team_model_add/delete calls required; public name is already registered +def _canonical_session_tags(tags: Sequence[AwsSessionTag]) -> tuple[tuple[str, str], ...]: + return tuple(sorted((tag["Key"], tag["Value"]) for tag in tags)) + + class ModelManagementAuthChecks: """ Common auth checks for model management endpoints @@ -1704,6 +1715,28 @@ class ModelManagementAuthChecks: param="litellm_credential_name", ) + @staticmethod + def can_user_set_aws_session_tags( + litellm_params: GenericLiteLLMParams | None, + user_api_key_dict: UserAPIKeyAuth, + existing_litellm_params: GenericLiteLLMParams | None = None, + ) -> Literal[True]: + if litellm_params is None or litellm_params.aws_session_tags is None: + return True + if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: + return True + existing_tags: Final = existing_litellm_params.aws_session_tags if existing_litellm_params is not None else None + if existing_tags is not None and _canonical_session_tags(existing_tags) == _canonical_session_tags( + litellm_params.aws_session_tags + ): + return True + raise ProxyException( + message=f"Only a proxy admin can set aws_session_tags on a model. Your role={user_api_key_dict.user_role}.", + type=ProxyErrorTypes.auth_error.value, + code=status.HTTP_403_FORBIDDEN, + param="aws_session_tags", + ) + @staticmethod async def allow_team_model_action( model_params: Deployment | updateDeployment, @@ -2037,6 +2070,11 @@ async def add_new_model( user_api_key_dict=user_api_key_dict, ) + ModelManagementAuthChecks.can_user_set_aws_session_tags( + litellm_params=model_params.litellm_params, + user_api_key_dict=user_api_key_dict, + ) + _raise_on_strategy_router_write_violation( incoming_params=model_params.litellm_params, existing_params=None, @@ -2221,6 +2259,12 @@ async def update_model( existing_litellm_params=deployment.litellm_params, ) + ModelManagementAuthChecks.can_user_set_aws_session_tags( + litellm_params=model_params.litellm_params, + user_api_key_dict=user_api_key_dict, + existing_litellm_params=deployment.litellm_params, + ) + _raise_on_strategy_router_write_violation( incoming_params=model_params.litellm_params, existing_params=deployment.litellm_params, diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index c02f886fc31..16dc1b9ff94 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -392,6 +392,185 @@ class TestModelManagementAuthChecks: assert exc_info.value.code == "403" mock_update.assert_not_awaited() + def test_can_user_set_aws_session_tags_admin_success(self): + result = ModelManagementAuthChecks.can_user_set_aws_session_tags( + litellm_params=LiteLLM_Params( + model="bedrock/test_model", aws_session_tags=[{"Key": "team", "Value": "genai"}] + ), + user_api_key_dict=self.admin_user, + ) + assert result is True + + def test_can_user_set_aws_session_tags_without_tags_allows_any_role(self): + result = ModelManagementAuthChecks.can_user_set_aws_session_tags( + litellm_params=LiteLLM_Params(model="bedrock/test_model", aws_role_name="arn:aws:iam::123:role/x"), + user_api_key_dict=self.team_admin_user, + ) + assert result is True + + def test_can_user_set_aws_session_tags_team_admin_fails(self): + with pytest.raises(Exception, match="Only a proxy admin can set aws_session_tags") as exc_info: + ModelManagementAuthChecks.can_user_set_aws_session_tags( + litellm_params=LiteLLM_Params( + model="bedrock/test_model", aws_session_tags=[{"Key": "team", "Value": "genai"}] + ), + user_api_key_dict=self.team_admin_user, + ) + assert exc_info.value.code == "403" + assert exc_info.value.param == "aws_session_tags" + + def test_can_user_set_aws_session_tags_unchanged_existing_allows_any_role(self): + result = ModelManagementAuthChecks.can_user_set_aws_session_tags( + litellm_params=LiteLLM_Params( + model="bedrock/test_model", + aws_session_tags=[{"Key": "team", "Value": "genai"}, {"Key": "env", "Value": "prod"}], + ), + user_api_key_dict=self.team_admin_user, + existing_litellm_params=LiteLLM_Params( + model="bedrock/test_model", + aws_session_tags=[{"Key": "env", "Value": "prod"}, {"Key": "team", "Value": "genai"}], + ), + ) + assert result is True + + def test_can_user_set_aws_session_tags_changed_value_fails_for_team_admin(self): + with pytest.raises(Exception, match="Only a proxy admin can set aws_session_tags") as exc_info: + ModelManagementAuthChecks.can_user_set_aws_session_tags( + litellm_params=LiteLLM_Params( + model="bedrock/test_model", aws_session_tags=[{"Key": "team", "Value": "platform"}] + ), + user_api_key_dict=self.team_admin_user, + existing_litellm_params=LiteLLM_Params( + model="bedrock/test_model", aws_session_tags=[{"Key": "team", "Value": "genai"}] + ), + ) + assert exc_info.value.code == "403" + + @pytest.mark.asyncio + async def test_add_new_model_rejects_aws_session_tags_for_non_admin(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + ) + + mock_prisma = MagicMock() + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch( # test-quality-ok: prior auth check needs a live DB; only the session tag check is under test + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + ): + with pytest.raises(ProxyException) as exc_info: + await add_new_model( + model_params=Deployment( + model_name="tagged-bedrock", + litellm_params=LiteLLM_Params( + model="bedrock/anthropic.claude-opus-4-6-v1:0", + aws_role_name="arn:aws:iam::123456789012:role/team-role", + aws_session_tags=[{"Key": "team", "Value": "genai"}], + ), + model_info={"id": "session-tags-create-test", "team_id": "test_team"}, + ), + user_api_key_dict=self.team_admin_user, + ) + assert exc_info.value.code == "403" + assert exc_info.value.param == "aws_session_tags" + mock_prisma.db.litellm_proxymodeltable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_patch_model_rejects_aws_session_tags_for_non_admin(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + patch_model, + ) + from litellm.types.router import updateLiteLLMParams + + model_id = "session-tags-patch-test" + db_model = Deployment( + model_name="tagged-bedrock", + litellm_params=LiteLLM_Params( + model="bedrock/anthropic.claude-opus-4-6-v1:0", + aws_role_name="arn:aws:iam::123456789012:role/team-role", + ), + model_info={"id": model_id, "team_id": "test_team"}, + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.llm_router", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch( # test-quality-ok: stubs the DB row fetch; only the session tag check is under test + "litellm.proxy.management_endpoints.model_management_endpoints.get_db_model", + new=AsyncMock(return_value=db_model), + ), + patch( # test-quality-ok: prior auth check needs a live DB; only the session tag check is under test + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + patch( # test-quality-ok: asserts the DB write is never reached on rejection + "litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db", + new=AsyncMock(), + ) as mock_update, + ): + with pytest.raises(ProxyException) as exc_info: + await patch_model( + model_id=model_id, + patch_data=updateDeployment( + litellm_params=updateLiteLLMParams(aws_session_tags=[{"Key": "team", "Value": "genai"}]) + ), + user_api_key_dict=self.team_admin_user, + ) + assert exc_info.value.code == "403" + assert exc_info.value.param == "aws_session_tags" + mock_update.assert_not_awaited() + + @pytest.mark.asyncio + async def test_update_model_rejects_aws_session_tags_for_non_admin(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_model, + ) + from litellm.types.router import updateLiteLLMParams + + model_id = "session-tags-put-test" + existing = Deployment( + model_name="tagged-bedrock", + litellm_params=LiteLLM_Params( + model="bedrock/anthropic.claude-opus-4-6-v1:0", + aws_role_name="arn:aws:iam::123456789012:role/team-role", + ), + model_info={"id": model_id, "team_id": "test_team"}, + ) + existing_row = MagicMock() + existing_row.model_dump.return_value = existing.model_dump() + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row) + mock_prisma.db.litellm_proxymodeltable.update = AsyncMock() + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.llm_router", None), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch( # test-quality-ok: prior auth check needs a live DB; only the session tag check is under test + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + ): + with pytest.raises(ProxyException) as exc_info: + await update_model( + model_params=updateDeployment( + litellm_params=updateLiteLLMParams(aws_session_tags=[{"Key": "team", "Value": "genai"}]), + model_info=ModelInfo(id=model_id), + ), + user_api_key_dict=self.team_admin_user, + ) + assert exc_info.value.code == "403" + assert exc_info.value.param == "aws_session_tags" + mock_prisma.db.litellm_proxymodeltable.update.assert_not_awaited() + def test_can_user_attach_credential_internal_user_fails(self): with pytest.raises(Exception, match="Only a proxy admin can attach a stored credential") as exc_info: ModelManagementAuthChecks.can_user_attach_credential( From 7e137958e93f91c21f954fabf878132a7534e311 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:12:40 -0700 Subject: [PATCH 245/411] fix(cli): let the apiKeyHelper supply Claude Code's key under lite claude `lite login --config-claude` writes an apiKeyHelper into ~/.claude/settings.json, and `lite claude` then also exported ANTHROPIC_AUTH_TOKEN, so Claude Code opened with its "Both ANTHROPIC_AUTH_TOKEN and apiKeyHelper set" banner and, because the env token wins, never ran the helper that was meant to refresh the key. When the launch key is the stored login key and settings.json carries exactly the helper lite wrote for this base URL, `lite claude` now leaves the token out of the env (dropping an inherited one) so Claude Code asks the helper. An explicit --api-key or LITELLM_PROXY_API_KEY, a helper for another proxy, a hand-written helper, or no settings file keep the previous behavior. --- litellm/proxy/client/cli/commands/agents.py | 29 ++++- .../client/cli/commands/claude_settings.py | 14 +++ .../proxy/client/cli/test_agents.py | 110 ++++++++++++++++++ .../proxy/client/cli/test_claude_settings.py | 43 +++++++ 4 files changed, 193 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index 7a0ae9dc955..8903be8ca5c 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -12,6 +12,7 @@ import requests from pydantic import BaseModel, TypeAdapter, ValidationError from .auth import CliContextObj, context_secret_vault, get_stored_api_key, login +from .claude_settings import CLAUDE_SETTINGS_PATH, lite_api_key_helper_configured from .cmd_quoting import quote_for_cmd from .pi import ( LITELLM_PROXY_API_KEY_ENV, @@ -83,6 +84,8 @@ def build_agent_env( base_url: str, api_key: str, profiles: frozenset[str], + *, + export_anthropic_token: bool = True, ) -> dict[str, str]: """Return a copy of base_env wired to route the agent through the proxy. @@ -97,12 +100,19 @@ def build_agent_env( proxy's /v1/models; likewise left alone when already set. pi ignores both base URL variables and instead resolves $LITELLM_PROXY_API_KEY from its synced models.json provider entry. + + With export_anthropic_token=False the bearer is left out (and any inherited + one dropped) so Claude Code asks its configured apiKeyHelper instead; Claude + Code prefers ANTHROPIC_AUTH_TOKEN over the helper and warns when both are set. """ env: Final = dict(base_env) root: Final = base_url.rstrip("/") if PROFILE_ANTHROPIC in profiles: env[ANTHROPIC_BASE_URL_ENV] = root - env[ANTHROPIC_AUTH_TOKEN_ENV] = api_key + if export_anthropic_token: + env[ANTHROPIC_AUTH_TOKEN_ENV] = api_key + else: + env.pop(ANTHROPIC_AUTH_TOKEN_ENV, None) env.pop(ANTHROPIC_API_KEY_ENV, None) if ENABLE_TOOL_SEARCH_ENV not in env: env[ENABLE_TOOL_SEARCH_ENV] = ENABLE_TOOL_SEARCH_VALUE @@ -460,6 +470,7 @@ def run_agent( launcher: Callable[[str, Sequence[str], Mapping[str, str]], None] = _hand_off, reattach_terminal: Callable[[], None] | None = None, preparers: Mapping[str, _Preparer] = MappingProxyType(_PREPARERS), + export_anthropic_token: bool = True, ) -> None: """Validate, wire the environment, and hand off to the agent. @@ -491,7 +502,9 @@ def run_agent( env: Final = MappingProxyType( { - **build_agent_env(env_before_sync, base_url, api_key, profiles), + **build_agent_env( + env_before_sync, base_url, api_key, profiles, export_anthropic_token=export_anthropic_token + ), **(_NO_EXTRA_ENV if isinstance(synced, ModelSyncSkipped) else synced), } ) @@ -529,14 +542,23 @@ def resolve_api_key(ctx: click.Context) -> str: _SKIP_VERIFY_HELP: Final = "Skip the pre-launch key check against the proxy." +def _helper_supplies_token(ctx_obj: CliContextObj, base_url: str, profiles: frozenset[str]) -> bool: + if PROFILE_ANTHROPIC not in profiles or not ctx_obj.get("api_key_from_token_file"): + return False + return lite_api_key_helper_configured(base_url, CLAUDE_SETTINGS_PATH) + + def _launch(ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify: bool) -> None: ctx_obj: Final[CliContextObj] = ctx.obj base_url: Final = ctx_obj["base_url"] started_interactive: Final = _is_interactive() api_key: Final = resolve_api_key(ctx) - display_name, _ = agent_profile(binary) + display_name, profiles = agent_profile(binary) + helper_supplies_token: Final = _helper_supplies_token(ctx_obj, base_url, profiles) click.echo(f"litellm: routing {display_name} through proxy at {base_url.rstrip('/')}") + if helper_supplies_token: + click.echo(f"litellm: {display_name} reads its key from the apiKeyHelper in {CLAUDE_SETTINGS_PATH}") try: run_agent( @@ -545,6 +567,7 @@ def _launch(ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify [binary, *args], skip_verify=skip_verify, reattach_terminal=(_restore_controlling_terminal if started_interactive else None), + export_anthropic_token=not helper_supplies_token, ) except AgentRunError as e: raise click.ClickException(str(e)) diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index 5e3ce95f088..bc58c2ed815 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -121,6 +121,19 @@ def resolve_api_key_helper(base_url: str, platform: str = sys.platform) -> str: return " ".join(quote(token) for token in (lite_path, "--base-url", base_url, "auth", "print-token")) +def lite_api_key_helper_configured(base_url: str, settings_path: Path) -> bool: + """Whether settings_path already carries the apiKeyHelper `lite login --config-claude` writes for base_url. + + Only an exact match counts: a helper for another proxy, a hand-written one, or + settings that cannot be read leave the caller on the env-token path. + """ + try: + configured_helper: Final = load_json_or_empty(settings_path).get(API_KEY_HELPER_KEY) + return configured_helper == resolve_api_key_helper(base_url.rstrip("/")) + except ClaudeSettingsError: + return False + + def write_claude_settings(base_url: str, settings_path: Path, owners: Sequence[SettingsFileOwner]) -> None: """Persistently point Claude Code at base_url, preserving every unrelated setting. @@ -169,6 +182,7 @@ __all__ = ( "SETTINGS_FILE_OWNERS", "ClaudeSettingsError", "SettingsFileOwner", + "lite_api_key_helper_configured", "load_json_or_empty", "merge_claude_settings", "resolve_api_key_helper", diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py index a8a6659fe9a..894c2406622 100644 --- a/tests/test_litellm/proxy/client/cli/test_agents.py +++ b/tests/test_litellm/proxy/client/cli/test_agents.py @@ -28,6 +28,7 @@ from litellm.proxy.client.cli.commands.agents import ( ) AGENTS_MODULE = "litellm.proxy.client.cli.commands.agents" +CLAUDE_SETTINGS_MODULE = "litellm.proxy.client.cli.commands.claude_settings" def _agent_command(name): @@ -163,6 +164,27 @@ class TestBuildAgentEnv: assert env["PATH"] == "/usr/bin" assert base == {"PATH": "/usr/bin", "ANTHROPIC_API_KEY": "real-key"} + def test_anthropic_profile_leaves_the_bearer_to_the_api_key_helper(self): + env = build_agent_env( + {"ANTHROPIC_AUTH_TOKEN": "stale-token", "ANTHROPIC_API_KEY": "real-key"}, + "http://localhost:4000/", + "sk-key", + frozenset({"anthropic"}), + export_anthropic_token=False, + ) + assert "ANTHROPIC_AUTH_TOKEN" not in env + assert "ANTHROPIC_API_KEY" not in env + assert env["ANTHROPIC_BASE_URL"] == "http://localhost:4000" + assert env["ENABLE_TOOL_SEARCH"] == "true" + assert env["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1" + + def test_helper_mode_still_exports_the_openai_key(self): + env = build_agent_env( + {}, "http://localhost:4000", "sk-key", frozenset({"anthropic", "openai"}), export_anthropic_token=False + ) + assert "ANTHROPIC_AUTH_TOKEN" not in env + assert env["OPENAI_API_KEY"] == "sk-key" + class TestAgentLaunchArgs: def test_claude_and_opencode_get_no_extra_args(self): @@ -509,6 +531,25 @@ class TestRunAgent: assert "ANTHROPIC_API_KEY" not in env assert "OPENAI_BASE_URL" not in env + def test_helper_supplied_token_never_reaches_the_launch_env(self): + calls = {} + verified = [] + + run_agent( + "http://localhost:4000", + "sk-key", + ["claude"], + base_env={"PATH": "/usr/bin", "ANTHROPIC_AUTH_TOKEN": "stale-token"}, + which=lambda name: "/usr/local/bin/claude", + verify=lambda base_url, api_key: verified.append(api_key), + launcher=lambda p, a, e: calls.update(env=dict(e)), + export_anthropic_token=False, + ) + + assert verified == ["sk-key"] + assert "ANTHROPIC_AUTH_TOKEN" not in calls["env"] + assert calls["env"]["ANTHROPIC_BASE_URL"] == "http://localhost:4000" + def test_codex_gets_openai_env(self): calls = {} run_agent( @@ -1052,6 +1093,75 @@ class TestAgentCommands: in result.output ) + def _invoke_claude_with_settings(self, tmp_path, settings, obj): + settings_path = tmp_path / "settings.json" + if settings is not None: + settings_path.write_text(json.dumps(settings)) + captured = {} + with ( + patch(f"{AGENTS_MODULE}.CLAUDE_SETTINGS_PATH", settings_path), + patch(f"{CLAUDE_SETTINGS_MODULE}.shutil.which", return_value="/usr/local/bin/lite"), + patch(f"{AGENTS_MODULE}.run_agent", side_effect=lambda b, k, c, **kw: captured.update(kw)), + ): + result = self.runner.invoke(_agent_command("claude"), [], obj=obj) + assert result.exit_code == 0, result.output + return captured, result.output + + def test_stored_login_with_a_matching_helper_leaves_the_token_to_the_helper(self, tmp_path): + captured, output = self._invoke_claude_with_settings( + tmp_path, + {"apiKeyHelper": "/usr/local/bin/lite --base-url http://localhost:4000 auth print-token"}, + {"base_url": "http://localhost:4000", "api_key": "sk-key", "api_key_from_token_file": True}, + ) + + assert captured["export_anthropic_token"] is False + assert "reads its key from the apiKeyHelper" in output + + def test_explicit_key_is_exported_even_when_a_helper_matches(self, tmp_path): + captured, output = self._invoke_claude_with_settings( + tmp_path, + {"apiKeyHelper": "/usr/local/bin/lite --base-url http://localhost:4000 auth print-token"}, + {"base_url": "http://localhost:4000", "api_key": "sk-key", "api_key_from_token_file": False}, + ) + + assert captured["export_anthropic_token"] is True + assert "apiKeyHelper" not in output + + def test_helper_for_another_proxy_keeps_the_env_token(self, tmp_path): + captured, _ = self._invoke_claude_with_settings( + tmp_path, + {"apiKeyHelper": "/usr/local/bin/lite --base-url https://other.example.com auth print-token"}, + {"base_url": "http://localhost:4000", "api_key": "sk-key", "api_key_from_token_file": True}, + ) + + assert captured["export_anthropic_token"] is True + + def test_no_claude_settings_keeps_the_env_token(self, tmp_path): + captured, _ = self._invoke_claude_with_settings( + tmp_path, + None, + {"base_url": "http://localhost:4000", "api_key": "sk-key", "api_key_from_token_file": True}, + ) + + assert captured["export_anthropic_token"] is True + + def test_codex_never_consults_claude_settings(self, tmp_path): + settings_path = tmp_path / "settings.json" + captured = {} + with ( + patch(f"{AGENTS_MODULE}.CLAUDE_SETTINGS_PATH", settings_path), + patch(f"{AGENTS_MODULE}.lite_api_key_helper_configured", side_effect=AssertionError("consulted")), + patch(f"{AGENTS_MODULE}.run_agent", side_effect=lambda b, k, c, **kw: captured.update(kw)), + ): + result = self.runner.invoke( + _agent_command("codex"), + [], + obj={"base_url": "http://localhost:4000", "api_key": "sk-key", "api_key_from_token_file": True}, + ) + + assert result.exit_code == 0, result.output + assert captured["export_anthropic_token"] is True + def test_codex_shows_friendly_name(self): captured = {} with patch( diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py index e5f2a9d95bd..8a2bfea3bd7 100644 --- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py +++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py @@ -15,6 +15,7 @@ from litellm.proxy.client.cli.commands.claude_settings import ( SETTINGS_FILE_OWNERS, ClaudeSettingsError, SettingsFileOwner, + lite_api_key_helper_configured, resolve_api_key_helper, write_claude_settings, ) @@ -357,3 +358,45 @@ class TestDoesNotDestroyUserOwnedStructure: write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) assert json.loads(settings_path.read_text())["env"] == "not-an-object" + + +class TestLiteApiKeyHelperConfigured: + def _settings(self, tmp_path, payload): + settings_path = tmp_path / "settings.json" + settings_path.write_text(payload) + return settings_path + + def test_recognises_the_helper_lite_login_wrote_for_this_proxy(self, tmp_path, lite_on_path): + settings_path = tmp_path / "settings.json" + write_claude_settings("https://proxy.example.com/", settings_path, ()) + + assert lite_api_key_helper_configured("https://proxy.example.com/", settings_path) is True + assert lite_api_key_helper_configured("https://proxy.example.com", settings_path) is True + + def test_a_helper_for_another_proxy_does_not_count(self, tmp_path, lite_on_path): + settings_path = tmp_path / "settings.json" + write_claude_settings("https://other.example.com", settings_path, ()) + + assert lite_api_key_helper_configured("https://proxy.example.com", settings_path) is False + + def test_a_hand_written_helper_does_not_count(self, tmp_path, lite_on_path): + settings_path = self._settings(tmp_path, json.dumps({"apiKeyHelper": "cat ~/.my-proxy-key"})) + + assert lite_api_key_helper_configured("https://proxy.example.com", settings_path) is False + + def test_missing_or_helperless_settings_do_not_count(self, tmp_path, lite_on_path): + assert lite_api_key_helper_configured("https://proxy.example.com", tmp_path / "absent.json") is False + helperless = json.dumps({"env": {"ANTHROPIC_BASE_URL": "https://proxy.example.com"}}) + settings_path = self._settings(tmp_path, helperless) + assert lite_api_key_helper_configured("https://proxy.example.com", settings_path) is False + + def test_unreadable_settings_fall_back_to_false(self, tmp_path, lite_on_path): + settings_path = self._settings(tmp_path, "{not json") + + assert lite_api_key_helper_configured("https://proxy.example.com", settings_path) is False + + def test_lite_missing_from_path_falls_back_to_false(self, tmp_path): + helper = "/usr/local/bin/lite --base-url https://proxy.example.com auth print-token" + settings_path = self._settings(tmp_path, json.dumps({"apiKeyHelper": helper})) + with patch(f"{CLAUDE_SETTINGS_MODULE}.shutil.which", return_value=None): + assert lite_api_key_helper_configured("https://proxy.example.com", settings_path) is False From e27a549aa919f82e590cf04513e72bf5a75f4a7d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:14:00 -0700 Subject: [PATCH 246/411] test(bedrock): type the probe credential overrides --- .../count_tokens/test_bedrock_count_tokens_handler.py | 6 +++++- .../test_litellm/llms/custom_httpx/test_llm_http_handler.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py b/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py index 31e65bb6e27..3622ce7f212 100644 --- a/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py +++ b/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock import httpx import pytest +from botocore.credentials import RefreshableCredentials from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -14,7 +15,10 @@ class _ProbedCountTokensHandler(BedrockCountTokensHandler): super().__init__() self._probe = probe - def get_credentials(self, **kwargs): + def get_credentials( + self, + **kwargs: object, # kwargs-ok: mirrors the base resolver's keyword contract, which the probe ignores + ) -> RefreshableCredentials: return self._probe.credentials() diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index b5eb98c4ce6..1140090f9cf 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, Mock, patch import httpx import pytest +from botocore.credentials import RefreshableCredentials import litellm from litellm._logging import verbose_logger @@ -822,7 +823,10 @@ class _ProbedBedrockMessagesConfig(AmazonAnthropicClaudeMessagesConfig): super().__init__() self._probe = probe - def get_credentials(self, **kwargs): + def get_credentials( + self, + **kwargs: object, # kwargs-ok: mirrors the base resolver's keyword contract, which the probe ignores + ) -> RefreshableCredentials: return self._probe.credentials() From 654afb547779eef5f7c575d4a84d24ea3f1163b0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:14:07 -0700 Subject: [PATCH 247/411] fix(e2e): assert the batch cost join on an in-run failed batch instead of a cross-run baton The batch list is served from LiteLLM_ManagedObjectTable whenever the managed files hook is loaded, and the Buildkite e2e stacks bundle a fresh Postgres per build, so a prior run's marker batch is never listed and the baton could only ever pass vacuously. Each run now creates a batch OpenAI fails at validation within seconds, retrieves it by its raw provider id with the same key until it is failed, and asserts the {provider_batch_id}_batch_cost row that retrieve writes joins the key's token hash and alias --- .../coverage_registry/quota_management.yaml | 2 +- .../test_key_attribution_e2e.py | 124 +++++++----------- 2 files changed, 45 insertions(+), 81 deletions(-) diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index d3966eed48f..c92fc0b43bd 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -61,4 +61,4 @@ - {id: quota_management.spend_tracking.key_attribution.joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [joins_key], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Every spend row a virtual key writes across chat, queued chat, messages, responses, embeddings, the Gemini passthrough, file upload, batch create, and a replayed callback log carries api_key equal to the key's token hash and the key alias, the join the usage APIs depend on; a re-hashed token shows up as an unattributed key-hash-* row (#39568, #39572)"} - {id: quota_management.spend_tracking.key_attribution.reports_alias_and_email, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [reports_alias_and_email], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/management_endpoints/internal_user_endpoints.py", rationale: "/spend/logs?api_key= returns every one of the key's rows with its alias and /user/daily/activity aggregates them under the key's token with key_alias and user_email; /spend/logs carries no email field, so the email is asserted on daily activity only"} - {id: quota_management.spend_tracking.key_attribution.health_rows_keep_service_account, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [health_rows_keep_service_account], exercised_on: [chat_completions], source: "proxy/health_check.py", rationale: "A /health probe's spend row stays keyed by the literal litellm-internal-health-check service account rather than a hash of it, so health spend never appears as an unattributed key"} -- {id: quota_management.spend_tracking.key_attribution.batch_cost_joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [batch_cost_joins_key], exercised_on: [batches], source: "proxy/batches_endpoints/endpoints.py", rationale: "Retrieving a completed batch by its raw provider id prices it inline against the retrieving key, so the {provider_batch_id}_batch_cost row must carry that key's token hash and alias; the target is the newest completed cross-run marker batch this proxy has not billed yet, since completion lags by up to 24h, and a cold start with no completed marker is a documented vacuous pass. The CheckBatchCost poller's own unified-id row needs the batch create row in the same database, which a stack booted fresh per run never holds for a completed marker"} +- {id: quota_management.spend_tracking.key_attribution.batch_cost_joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [batch_cost_joins_key], exercised_on: [batches], source: "proxy/batches_endpoints/endpoints.py", rationale: "The retrieve that first sees a batch in a terminal state writes its {provider_batch_id}_batch_cost row, so the batch each run creates is one OpenAI fails at validation within seconds and the test retrieves it by its raw provider id with the same key until it is failed; that raw-id retrieve is never owned by the CheckBatchCost poller, so it prices the batch inline against the retrieving key and the row must carry that key's token hash and alias. A completed batch with a positive cost is out of one run's reach: OpenAI's completion window is 24h and a stack booted fresh per run lists no earlier run's batches"} diff --git a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py index cac4fd7fb3c..1d7b5e128f5 100644 --- a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py @@ -12,19 +12,17 @@ re-hashed token (v1.99.0's regression, #39568 and #39572) shows up as a key-hash-* row with no alias and no email in the customer's usage exports. The health-check service account writes rows too; those must stay keyed by the -literal service-account name, never by a hash of it. A batch's cost row lands -only once the batch completes, up to 24h later, so the batch cost path rides a -cross-run baton like the batches suite: each run submits a one-line marker batch -and never cancels it, and the newest completed marker from any run that this -proxy has not billed yet is retrieved by its raw provider id with this run's -key. That retrieve is the writer under test: a raw id is never poller-owned, so -the proxy prices it inline against the retrieving key, and the -{provider_batch_id}_batch_cost row must join this run's token with its alias. -The CheckBatchCost poller's own row (the unified id, billed against the -submitting key) needs the batch's create row in the same database, which a stack -booted fresh per run never holds for a completed marker, so that writer is out -of this test's reach. A cold start with no completed marker is a documented -vacuous pass, never a skip. +literal service-account name, never by a hash of it. A batch's cost row is +written by the retrieve that first sees the batch in a terminal state, so the +batch the run creates is one OpenAI fails at validation within seconds (its one +line targets /v1/embeddings under a /v1/chat/completions batch), and the test +retrieves it by its raw provider id with the same key until it is failed. A raw +id is never owned by the CheckBatchCost poller, so that retrieve prices the batch +inline against the retrieving key and its {provider_batch_id}_batch_cost row +must join the key's token with its alias. A completed batch with a positive +cost is out of a single run's reach (OpenAI's completion window is 24h, and a +stack booted fresh per run lists no earlier run's batches), so the poller's own +row is not asserted here. /spend/logs carries no email field, so the email assertion lives on /user/daily/activity alone; /spend/logs is held to the alias in metadata. @@ -38,7 +36,7 @@ from datetime import datetime, timedelta, timezone from typing import Final import pytest -from models import ChatMessage, KeyGenerateBody, SpendLogsParams +from models import KeyGenerateBody from proxy_client import Converged, await_converged from pydantic import BaseModel from spend_e2e_client import ( @@ -64,11 +62,9 @@ BATCH_MODEL: Final = "openai-gpt-4o-mini" BATCH_BACKEND_MODEL: Final = "gpt-4o-mini" BATCH_PROVIDER: Final = "openai" HEALTH_SERVICE_ACCOUNT: Final = "litellm-internal-health-check" -BATON_MARKER_KEY: Final = "litellm_e2e_suite" -BATON_MARKER_VALUE: Final = "key-attribution-baton" -BATON_POLL_SECONDS: Final = 30.0 -BATON_POLL_INTERVAL_SECONDS: Final = 10.0 -BATON_LIST_LIMIT: Final = 100 +BATCH_TERMINAL_STATUSES: Final = frozenset({"completed", "failed", "cancelled", "expired"}) +FAILED_BATCH_POLL_SECONDS: Final = 120.0 +FAILED_BATCH_POLL_INTERVAL_SECONDS: Final = 5.0 MAX_TOKENS: Final = 8 REPLAY_RESPONSE_COST: Final = 0.0001 REPLAY_PROMPT_TOKENS: Final = 5 @@ -86,17 +82,16 @@ WRITE_PATHS: Final = ( ) -class BatchLineBody(BaseModel): +class EmbeddingLineBody(BaseModel): model: str - messages: list[ChatMessage] - max_tokens: int + input: str -class BatchLine(BaseModel): +class EmbeddingLine(BaseModel): custom_id: str method: str = "POST" - url: str = "/v1/chat/completions" - body: BatchLineBody + url: str = "/v1/embeddings" + body: EmbeddingLineBody @dataclass(frozen=True, slots=True) @@ -134,26 +129,19 @@ def _call_id(name: str, sent: StreamingResponse) -> WritePath: return WritePath(name=name, request_id=sent.call_id) -def _batch_jsonl(marker: str) -> bytes: - line: Final = BatchLine( - custom_id=marker, - body=BatchLineBody( - model=BATCH_BACKEND_MODEL, - messages=[ChatMessage(role="user", content=f"Reply with the word ok. {marker}")], - max_tokens=MAX_TOKENS, - ), - ) +def _endpoint_mismatched_jsonl(marker: str) -> bytes: + line: Final = EmbeddingLine(custom_id=marker, body=EmbeddingLineBody(model=BATCH_BACKEND_MODEL, input=marker)) return f"{line.model_dump_json()}\n".encode() def _drive_batch(client: SpendClient, identity: AttributedKey, marker: str) -> tuple[WritePath, WritePath]: - uploaded: Final = client.upload_batch_file(identity.key, BATCH_MODEL, _batch_jsonl(marker)) + uploaded: Final = client.upload_batch_file(identity.key, BATCH_MODEL, _endpoint_mismatched_jsonl(marker)) created: Final = client.create_batch( identity.key, BatchCreateBody( input_file_id=uploaded.id, model=BATCH_MODEL, - metadata={BATON_MARKER_KEY: BATON_MARKER_VALUE, "run": marker}, + metadata={"run": marker}, ), ) return ( @@ -204,48 +192,26 @@ def _drive_every_write_path(client: SpendClient, identity: AttributedKey) -> tup ) -def _completed_batons(listed: list[BatchObject]) -> tuple[BatchObject, ...]: - return tuple( - sorted( - ( - batch - for batch in listed - if batch.status == "completed" and (batch.metadata or {}).get(BATON_MARKER_KEY) == BATON_MARKER_VALUE - ), - key=lambda batch: batch.created_at or 0, - reverse=True, - ) - ) - - def _provider_batch_id(unified_batch_id: str) -> str: encoded: Final = unified_batch_id.removeprefix("batch_") decoded: Final = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)).decode() return decoded.removeprefix("litellm:").split(";", 1)[0] -def _batch_cost_request_id(unified_batch_id: str) -> str: - return f"{_provider_batch_id(unified_batch_id)}_batch_cost" +def _driven_batch_id(driven: DrivenKey) -> str: + return next(path.request_id for path in driven.paths if path.name == "batch_create") -def _newest_unbilled_completed_baton(client: SpendClient, key: str) -> BatchObject | None: +def _await_terminal_batch(client: SpendClient, key: str, provider_batch_id: str) -> BatchObject: outcome: Final = await_converged( - lambda: _completed_batons(client.list_batches(key, BATCH_MODEL, limit=BATON_LIST_LIMIT)), - converged=lambda completed: bool(completed), - timeout=BATON_POLL_SECONDS, - interval=BATON_POLL_INTERVAL_SECONDS, + lambda: client.retrieve_batch(key, provider_batch_id, provider=BATCH_PROVIDER), + converged=lambda batch: batch.status in BATCH_TERMINAL_STATUSES, + timeout=FAILED_BATCH_POLL_SECONDS, + interval=FAILED_BATCH_POLL_INTERVAL_SECONDS, now=time.monotonic, sleep=time.sleep, ) - completed: Final = outcome.result if isinstance(outcome, Converged) else () - return next( - ( - batch - for batch in completed - if not client.proxy.spend_logs(SpendLogsParams(request_id=_batch_cost_request_id(batch.id))) - ), - None, - ) + return outcome.result if isinstance(outcome, Converged) else outcome.last_result def _health_rows_between(client: SpendClient, started_at: datetime) -> list[SpendLogRow]: @@ -416,23 +382,21 @@ class TestKeyAttribution: "quota_management.spend_tracking.key_attribution.batch_cost_joins_key", exercised_on=["batches"], ) - def test_completed_batch_cost_row_joins_the_key(self, client: SpendClient, driven: DrivenKey) -> None: - completed: Final = _newest_unbilled_completed_baton(client, driven.identity.key) - if completed is None: - return - provider_batch_id: Final = _provider_batch_id(completed.id) - fetched: Final = client.retrieve_batch(driven.identity.key, provider_batch_id, provider=BATCH_PROVIDER) - assert fetched.status == "completed", f"listed-completed marker retrieved as {fetched.status!r}" - cost_request_id: Final = _batch_cost_request_id(completed.id) - rows: Final = client.proxy.poll_logs_for_request_id( - cost_request_id, - predicate=lambda found: any((row.spend or 0) > 0 for row in found), + def test_terminal_batch_cost_row_joins_the_retrieving_key(self, client: SpendClient, driven: DrivenKey) -> None: + provider_batch_id: Final = _provider_batch_id(_driven_batch_id(driven)) + fetched: Final = _await_terminal_batch(client, driven.identity.key, provider_batch_id) + assert fetched.status == "failed", ( + f"endpoint-mismatched batch {provider_batch_id} is {fetched.status!r} after " + f"{FAILED_BATCH_POLL_SECONDS:.0f}s, so its terminal cost row cannot be asserted" ) - priced: Final = [row for row in rows if (row.spend or 0) > 0] - assert priced, f"retrieving completed batch {provider_batch_id} wrote no positive-cost row under {cost_request_id}" + cost_request_id: Final = f"{provider_batch_id}_batch_cost" + rows: Final = client.proxy.poll_logs_for_request_id(cost_request_id) + assert rows, f"retrieving failed batch {provider_batch_id} wrote no cost row under {cost_request_id}" + call_types: Final = tuple(sorted({row.call_type or "" for row in rows})) + assert call_types == ("aretrieve_batch",), f"cost rows under {cost_request_id} carry call types {call_types}" unjoined: Final = [ (row.call_type, row.api_key, row.metadata.user_api_key_alias if row.metadata else None) - for row in priced + for row in rows if row.api_key != driven.identity.token or row.metadata is None or row.metadata.user_api_key_alias != driven.identity.alias From a248ff2c6ea712b0f8cb9f03bc327701c8dd63ab Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:14:50 -0700 Subject: [PATCH 248/411] fix(guardrails): fail open on Responses tool-call rewrites in another shape and keep item names Guardrails that hand back tool_calls in their own shape (vendor JSON, user code output) raised a KeyError on the non-stream Responses write-back. Returned tool calls are now validated before comparison; a shape or count that does not line up leaves every tool-call item unchanged and logs a warning naming the guardrail. A tool call's name is written back only when the guardrail changed it, so a nameless custom_tool_call no longer picks up the custom_tool placeholder. --- .../guardrail_translation/handler.py | 85 ++++++++++++++---- ...test_openai_responses_guardrail_handler.py | 90 +++++++++++++++++++ 2 files changed, 159 insertions(+), 16 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index fd0d7b66bee..8c8294ad2fc 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -37,7 +37,7 @@ from itertools import accumulate, chain, repeat from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -105,6 +105,19 @@ class _ToolCallShape(NamedTuple): arguments: str +class _ToolCallFunctionFields(BaseModel): + model_config = ConfigDict(frozen=True) + + name: str | None = None + arguments: str = "" + + +class _ToolCallFields(BaseModel): + model_config = ConfigDict(frozen=True) + + function: _ToolCallFunctionFields + + def _tool_call_shapes(tool_calls: Sequence[ChatCompletionToolCallChunk]) -> tuple[_ToolCallShape, ...]: return tuple( _ToolCallShape(name=tool_call["function"].get("name"), arguments=tool_call["function"].get("arguments", "")) @@ -112,6 +125,47 @@ def _tool_call_shapes(tool_calls: Sequence[ChatCompletionToolCallChunk]) -> tupl ) +def _returned_tool_call_shape(tool_call: object) -> _ToolCallShape | None: + payload: Final = tool_call.model_dump() if isinstance(tool_call, BaseModel) else tool_call + try: + fields: Final = _ToolCallFields.model_validate(payload) + except ValidationError: + return None + return _ToolCallShape(name=fields.function.name, arguments=fields.function.arguments) + + +def _post_guardrail_tool_call_shapes( + returned_tool_calls: Sequence[object] | None, + pre_guardrail_tool_calls: tuple[_ToolCallShape, ...], + guardrail_name: str | None, +) -> tuple[_ToolCallShape, ...]: + if not pre_guardrail_tool_calls: + return pre_guardrail_tool_calls + if returned_tool_calls is None or len(returned_tool_calls) != len(pre_guardrail_tool_calls): + verbose_proxy_logger.warning( + "OpenAI Responses API: guardrail %s returned %s tool calls for the %d scanned, " + "leaving the tool call output items unchanged", + guardrail_name, + "no" if returned_tool_calls is None else len(returned_tool_calls), + len(pre_guardrail_tool_calls), + ) + return pre_guardrail_tool_calls + returned_shapes: Final = tuple(_returned_tool_call_shape(tool_call) for tool_call in returned_tool_calls) + validated_shapes: Final = tuple(shape for shape in returned_shapes if shape is not None) + if len(validated_shapes) != len(returned_shapes): + verbose_proxy_logger.warning( + "OpenAI Responses API: guardrail %s returned tool calls without a function name and arguments, " + "leaving the tool call output items unchanged", + guardrail_name, + ) + return pre_guardrail_tool_calls + return validated_shapes + + +def _tool_call_rewrite(before: _ToolCallShape, after: _ToolCallShape) -> _ToolCallShape: + return _ToolCallShape(name=after.name if after.name != before.name else None, arguments=after.arguments) + + class ResponseOutputEnvelope(TypedDict, total=False): """Dict form of a Responses API response, as far as guardrail write-back reads it.""" @@ -698,7 +752,11 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts: Final = guardrailed_inputs.get("texts", []) - returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") + post_guardrail_tool_calls: Final = _post_guardrail_tool_call_shapes( + returned_tool_calls=guardrailed_inputs.get("tool_calls"), + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + guardrail_name=guardrail_to_apply.guardrail_name, + ) # Step 3: Map guardrail responses back to original response structure await self._apply_guardrail_responses_to_output( @@ -709,11 +767,7 @@ class OpenAIResponsesHandler(BaseTranslation): self._write_tool_call_rewrites_to_output( tool_call_items=tuple(item for item in response_output if _is_tool_call_output_item(item)), pre_guardrail_tool_calls=pre_guardrail_tool_calls, - post_guardrail_tool_calls=_tool_call_shapes( - returned_tool_calls - if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) - else tool_calls_to_check - ), + post_guardrail_tool_calls=post_guardrail_tool_calls, ) verbose_proxy_logger.debug("OpenAI Responses API: Processed output response: %s", response) @@ -811,11 +865,10 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts: Final = guardrailed_inputs.get("texts", []) - returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") - post_guardrail_tool_calls: Final = _tool_call_shapes( - returned_tool_calls - if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) - else tool_calls_to_check + post_guardrail_tool_calls: Final = _post_guardrail_tool_call_shapes( + returned_tool_calls=guardrailed_inputs.get("tool_calls"), + pre_guardrail_tool_calls=pre_guardrail_tool_calls, + guardrail_name=guardrail_to_apply.guardrail_name, ) # Write guardrailed texts back into the output items in-place. @@ -991,7 +1044,7 @@ class OpenAIResponsesHandler(BaseTranslation): ) rewrites_by_call_id: Final = MappingProxyType( { - call_id: after + call_id: _tool_call_rewrite(before, after) for call_id, before, after in zip(call_ids, pre_guardrail_tool_calls, post_guardrail_tool_calls) if after != before } @@ -1050,12 +1103,12 @@ class OpenAIResponsesHandler(BaseTranslation): ) -> None: if len(tool_call_items) != len(post_guardrail_tool_calls): return - for output_item, after in ( - (output_item, after) + for output_item, rewrite in ( + (output_item, _tool_call_rewrite(before, after)) for output_item, before, after in zip(tool_call_items, pre_guardrail_tool_calls, post_guardrail_tool_calls) if after != before ): - self._write_tool_call_item(output_item, after.name, after.arguments) + self._write_tool_call_item(output_item, rewrite.name, rewrite.arguments) @staticmethod def _tool_call_ids_by_item_id(stream_events: Sequence[object]) -> Mapping[str, str]: diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index cf4895f637f..a4f0a77a9b6 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -10,6 +10,8 @@ from collections.abc import Callable from typing import Any, List, Literal, Optional, Tuple from unittest.mock import AsyncMock, MagicMock +import logging + import pytest @@ -85,6 +87,29 @@ class PersimmonMaskingGuardrail(CustomGuardrail): return {**inputs, "tool_calls": tool_calls} +class FlatShapeGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[LiteLLMLoggingObj] = None, + ) -> GenericGuardrailAPIInputs: + flat_tool_calls = [{"name": "exec", "input": "rm -rf /"} for _ in inputs.get("tool_calls", [])] + return {**inputs, "tool_calls": flat_tool_calls} + + +class DroppingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[LiteLLMLoggingObj] = None, + ) -> GenericGuardrailAPIInputs: + return {**inputs, "tool_calls": []} + + CUSTOM_TOOL_CALL_ITEM = { "type": "custom_tool_call", "id": "ctc_1", @@ -737,6 +762,52 @@ class TestOpenAIResponsesHandlerToolCallExtraction: assert (custom_item.name if typed else custom_item["name"]) == "exec" assert (output[0].content[0].text if typed else output[0]["content"][0]["text"]) == "running persimmon" + @staticmethod + def _custom_tool_call_response(item: dict) -> dict: + return { + "id": "resp_1", + "created_at": 1, + "model": "gpt-5.6", + "object": "response", + "status": "completed", + "output": [item], + } + + @pytest.mark.asyncio + async def test_process_output_response_ignores_tool_call_rewrites_in_another_shape(self): + handler = OpenAIResponsesHandler() + response = self._custom_tool_call_response(dict(CUSTOM_TOOL_CALL_ITEM)) + + result = await handler.process_output_response(response, FlatShapeGuardrail(guardrail_name="flat")) + + assert result["output"][0]["input"] == "echo persimmon" + assert result["output"][0]["name"] == "exec" + + @pytest.mark.asyncio + async def test_process_output_response_warns_when_guardrail_drops_tool_calls(self, caplog): + handler = OpenAIResponsesHandler() + response = self._custom_tool_call_response(dict(CUSTOM_TOOL_CALL_ITEM)) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result = await handler.process_output_response(response, DroppingGuardrail(guardrail_name="dropper")) + + assert result["output"][0]["input"] == "echo persimmon" + assert any( + "dropper" in record.getMessage() and "0 tool calls for the 1 scanned" in record.getMessage() + for record in caplog.records + ) + + @pytest.mark.asyncio + async def test_process_output_response_keeps_a_nameless_custom_tool_call_nameless(self): + handler = OpenAIResponsesHandler() + nameless_item = {key: value for key, value in CUSTOM_TOOL_CALL_ITEM.items() if key != "name"} + response = self._custom_tool_call_response(nameless_item) + + result = await handler.process_output_response(response, PersimmonMaskingGuardrail(guardrail_name="mask")) + + assert result["output"][0]["input"] == "echo [MASKED]" + assert "name" not in result["output"][0] + @pytest.mark.asyncio async def test_process_output_response_with_tool_calls(self): """Test processing output response containing function tool calls""" @@ -1469,6 +1540,25 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert events[5]["response"]["output"][0]["name"] == "exec" assert "arguments" not in events[5]["response"]["output"][0] + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_keep_a_nameless_custom_tool_call_nameless(self): + handler = OpenAIResponsesHandler() + events = self._ended_custom_tool_call_stream_events() + items = [events[0]["item"], events[4]["item"], events[5]["response"]["output"][0]] + for item in items: + del item["name"] + + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=PersimmonMaskingGuardrail(guardrail_name="mask"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert events[3]["input"] == "echo [MASKED]" + assert events[5]["response"]["output"][0]["input"] == "echo [MASKED]" + assert all("name" not in item for item in items) + @pytest.mark.asyncio async def test_deliver_ended_stream_rewrites_syncs_typed_custom_tool_call_events(self): from litellm.types.llms.openai import ( From c972bbe80fb575fb0a162e79547edbe1db22298d Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:22:41 -0700 Subject: [PATCH 249/411] fix(mcp): send debug headers immediately for GET streams --- .../proxy/_experimental/mcp_server/mcp_debug.py | 8 ++++++-- litellm/proxy/_experimental/mcp_server/server.py | 4 +++- .../_experimental/mcp_server/test_mcp_debug.py | 15 ++++++++++----- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_debug.py b/litellm/proxy/_experimental/mcp_server/mcp_debug.py index b25ed27becc..ff30ef99ebd 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_debug.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_debug.py @@ -308,14 +308,18 @@ class MCPDebug: @staticmethod def wrap_send_with_debug_headers( - send: Send, debug_headers: Mapping[str, str], resolution: Callable[[], Mapping[str, str]] | None = None + send: Send, + debug_headers: Mapping[str, str], + resolution: Callable[[], Mapping[str, str]] | None = None, + *, + request_method: str | None = None, ) -> Send: """ Return a new ASGI ``send`` callable that injects *debug_headers* into the ``http.response.start`` message. """ - if resolution is not None: + if resolution is not None and request_method == "POST": return _DiagnosticSend(send, debug_headers, resolution) async def _send_with_debug(message: Message) -> None: diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index b53c55ef5fa..72d256e90fc 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -4482,7 +4482,9 @@ if MCP_AVAILABLE: diagnostics: Final = MCPAuthDiagnostics() if _debug_headers else None if diagnostics is not None: scope[MCP_AUTH_DIAGNOSTICS_SCOPE_KEY] = diagnostics - send = MCPDebug.wrap_send_with_debug_headers(send, _debug_headers, diagnostics.headers) + send = MCPDebug.wrap_send_with_debug_headers( + send, _debug_headers, diagnostics.headers, request_method=scope.get("method") + ) # Ensure session managers are initialized if not _SESSION_MANAGERS_INITIALIZED: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_debug.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_debug.py index 5039209c351..7d46bb0237a 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_debug.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_debug.py @@ -208,20 +208,25 @@ class TestWrapSendWithDebugHeaders: @pytest.mark.asyncio @pytest.mark.parametrize("source", tuple(AuthResolution)) -async def test_debug_uses_resolution_recorded_after_response_start(source: AuthResolution) -> None: +@pytest.mark.parametrize("method", ("GET", "DELETE", "POST")) +async def test_debug_defers_resolution_until_first_frame_only_for_post(source: AuthResolution, method: str) -> None: captured: Final[list[Message]] = [] diagnostics: Final = MCPAuthDiagnostics() async def send(message: Message) -> None: captured.append(message) - wrapped: Final = MCPDebug.wrap_send_with_debug_headers(send, {}, diagnostics.headers) + wrapped: Final = MCPDebug.wrap_send_with_debug_headers( + send, diagnostics.headers(), diagnostics.headers, request_method=method + ) await wrapped({"type": "http.response.start", "status": 200, "headers": []}) - assert captured == [] + assert len(captured) == (0 if method == "POST" else 1) diagnostics.record("s1", source) body: Final[Message] = {"type": "http.response.body", "body": b"data: pong\n\n", "more_body": True} await wrapped(body) - assert dict(captured[0]["headers"])[b"x-mcp-debug-auth-resolution"] == source.value.encode() + assert dict(captured[0]["headers"])[b"x-mcp-debug-auth-resolution"] == ( + source.value.encode() if method == "POST" else b"unresolved" + ) assert captured[1] == body @@ -233,7 +238,7 @@ async def test_early_stream_frame_reports_unresolved_without_waiting() -> None: async def send(message: Message) -> None: captured.append(message) - wrapped: Final = MCPDebug.wrap_send_with_debug_headers(send, {}, diagnostics.headers) + wrapped: Final = MCPDebug.wrap_send_with_debug_headers(send, {}, diagnostics.headers, request_method="POST") await wrapped({"type": "http.response.start", "status": 200, "headers": []}) await wrapped({"type": "http.response.body", "body": b": ping\n\n", "more_body": True}) diagnostics.record("s1", AuthResolution.stored_user_token) From 5db99ca7ba111c19d7b3aedd434abad436108ddf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:23:45 -0700 Subject: [PATCH 250/411] test(e2e): register the poller batch cost join as its own uncovered cell The retrieve writer and the CheckBatchCost poller are two different spend writers, and the in-run failed batch only proves the first. Split the key_attribution batch cell in two: retrieve_batch_cost_joins_retrieving_key, which test_terminal_batch_cost_row_joins_the_retrieving_key claims, and poller_batch_cost_joins_creating_key, which no test claims yet and so shows up on the coverage dashboard as a P1 gap instead of hiding behind the retrieve leg. The rationale records why one run cannot hand the poller a completed batch on a stack that boots a fresh Postgres per build. --- tests/e2e/CLAUDE.md | 3 ++- tests/e2e/coverage_registry/quota_management.yaml | 3 ++- .../spend_tracking/test_key_attribution_e2e.py | 6 ++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index b92d84b8b25..34fbe9d9247 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -188,7 +188,8 @@ quota_management... | routes_to_fallback | reseed_matches_db | reports_spend | logs_cost | zero_cost | matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows | writes_failure_row | returns_cost | keeps_total | joins_key | reports_alias_and_email - | health_rows_keep_service_account | batch_cost_joins_key + | health_rows_keep_service_account | retrieve_batch_cost_joins_retrieving_key + | poller_batch_cost_joins_creating_key e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages] quota_management.budget.key.blocks_over_limit exercised_on=[chat_completions] ``` diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index c92fc0b43bd..ad0914d455b 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -61,4 +61,5 @@ - {id: quota_management.spend_tracking.key_attribution.joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [joins_key], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Every spend row a virtual key writes across chat, queued chat, messages, responses, embeddings, the Gemini passthrough, file upload, batch create, and a replayed callback log carries api_key equal to the key's token hash and the key alias, the join the usage APIs depend on; a re-hashed token shows up as an unattributed key-hash-* row (#39568, #39572)"} - {id: quota_management.spend_tracking.key_attribution.reports_alias_and_email, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [reports_alias_and_email], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/management_endpoints/internal_user_endpoints.py", rationale: "/spend/logs?api_key= returns every one of the key's rows with its alias and /user/daily/activity aggregates them under the key's token with key_alias and user_email; /spend/logs carries no email field, so the email is asserted on daily activity only"} - {id: quota_management.spend_tracking.key_attribution.health_rows_keep_service_account, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [health_rows_keep_service_account], exercised_on: [chat_completions], source: "proxy/health_check.py", rationale: "A /health probe's spend row stays keyed by the literal litellm-internal-health-check service account rather than a hash of it, so health spend never appears as an unattributed key"} -- {id: quota_management.spend_tracking.key_attribution.batch_cost_joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [batch_cost_joins_key], exercised_on: [batches], source: "proxy/batches_endpoints/endpoints.py", rationale: "The retrieve that first sees a batch in a terminal state writes its {provider_batch_id}_batch_cost row, so the batch each run creates is one OpenAI fails at validation within seconds and the test retrieves it by its raw provider id with the same key until it is failed; that raw-id retrieve is never owned by the CheckBatchCost poller, so it prices the batch inline against the retrieving key and the row must carry that key's token hash and alias. A completed batch with a positive cost is out of one run's reach: OpenAI's completion window is 24h and a stack booted fresh per run lists no earlier run's batches"} +- {id: quota_management.spend_tracking.key_attribution.retrieve_batch_cost_joins_retrieving_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [retrieve_batch_cost_joins_retrieving_key], exercised_on: [batches], source: "proxy/batches_endpoints/endpoints.py", rationale: "The retrieve that first sees a batch in a terminal state prices it inline and writes its {provider_batch_id}_batch_cost row against the retrieving key, so the batch each run creates is one OpenAI fails at validation within seconds and the test retrieves it by its raw provider id with the same key until it is failed; a raw id is never owned by the CheckBatchCost poller, and the row must carry that key's token hash and alias"} +- {id: quota_management.spend_tracking.key_attribution.poller_batch_cost_joins_creating_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [poller_batch_cost_joins_creating_key], exercised_on: [batches], source: "enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py", rationale: "The CheckBatchCost poller bills a completed, positive-cost batch created through a unified id against the key that created it, a different writer from the inline retrieve. No test claims this cell yet: OpenAI's completion window is 24h and both e2e stacks boot a fresh Postgres per build, so a completed batch is out of one run's reach and the managed list never shows an earlier run's batch; the cell stays visible as a gap until a run can hand a completed batch to the poller"} diff --git a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py index 1d7b5e128f5..4a2c23927c6 100644 --- a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py @@ -283,9 +283,7 @@ class TestKeyAttribution: ) def test_every_write_path_row_joins_the_key(self, client: SpendClient, driven: DrivenKey) -> None: assert tuple(path.name for path in driven.paths) == WRITE_PATHS - found: Final = tuple( - (path, client.proxy.poll_logs_for_request_id(path.request_id)) for path in driven.paths - ) + found: Final = tuple((path, client.proxy.poll_logs_for_request_id(path.request_id)) for path in driven.paths) unwritten: Final = [path.name for path, rows in found if not rows] assert not unwritten, f"write paths that produced no spend row within the poll window: {unwritten}" unjoined: Final = [ @@ -379,7 +377,7 @@ class TestKeyAttribution: assert not rehashed, f"health-check rows keyed by something other than {HEALTH_SERVICE_ACCOUNT!r}: {rehashed}" @pytest.mark.covers( - "quota_management.spend_tracking.key_attribution.batch_cost_joins_key", + "quota_management.spend_tracking.key_attribution.retrieve_batch_cost_joins_retrieving_key", exercised_on=["batches"], ) def test_terminal_batch_cost_row_joins_the_retrieving_key(self, client: SpendClient, driven: DrivenKey) -> None: From e27a018a6ce1ecf0fdbfa071da8e50d5b9460d96 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:29:52 -0700 Subject: [PATCH 251/411] fix(router): pin bridge-replayed encrypted reasoning to the deployment that minted it The encrypted_content_affinity check only read the pin from the Responses input, which /v1/messages builds after the router has picked a deployment, so a model group spread across OpenAI orgs sent follow-up turns to the wrong org and got invalid_encrypted_content back. The check now also decodes the pin from bridge-tagged thinking and redacted_thinking blocks in the Anthropic messages. The bridge also keeps a deployment's own include list next to reasoning.encrypted_content instead of replacing it. --- .../prompt_templates/common_utils.py | 4 +- .../responses_adapters/handler.py | 10 +++- .../encrypted_content_affinity_check.py | 50 ++++++++++++++++--- .../test_responses_adapters_handler.py | 10 ++++ .../test_encrypted_content_affinity_check.py | 40 +++++++++++++++ 5 files changed, 104 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 75a3aefa98b..ae9b7f6bc4b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1873,7 +1873,7 @@ def _encrypted_reasoning_field(block: Mapping[str, object]) -> object: return None -def _encrypted_content_of_block(block: Mapping[str, object]) -> str | None: +def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: return encrypted_content_from_signature(_encrypted_reasoning_field(block)) @@ -1900,7 +1900,7 @@ def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> for block in group if (text := _readable_thinking_text(block)) ] - encrypted_content: Final = _encrypted_content_of_block(group[0]) + encrypted_content: Final = encrypted_content_of_block(group[0]) if encrypted_content is not None: return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content) if not summary: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 24825cc87e0..7731c883d9f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -126,7 +126,7 @@ def _build_responses_kwargs( responses_kwargs["stream"] = True # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) - excluded: Final = {"anthropic_messages"} + excluded: Final = frozenset(("anthropic_messages",)) for key, value in forwarded_kwargs.items(): if key == "litellm_logging_obj" and value is not None: from litellm.litellm_core_utils.litellm_logging import ( @@ -147,6 +147,14 @@ def _build_responses_kwargs( if explicit_prompt_cache_key is not None: responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key + deployment_include: Final = forwarded_kwargs.get("include") + bridge_include: Final = responses_kwargs.get("include") + if isinstance(deployment_include, list) and isinstance(bridge_include, list): + responses_kwargs["include"] = [ + *bridge_include, + *(item for item in deployment_include if item not in bridge_include), + ] + return responses_kwargs diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index b623e31ce06..ba94bd86a56 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -37,6 +37,7 @@ Safe to enable globally: """ import time +from collections.abc import Iterator, Mapping from typing import TYPE_CHECKING, Final, Optional, Protocol, cast import httpx @@ -48,6 +49,7 @@ from litellm.exceptions import ( ServiceUnavailableError, ) from litellm.integrations.custom_logger import CustomLogger, Span +from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_content_of_block from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.cooldown_cache import CooldownCacheValue from litellm.types.llms.openai import AllMessageValues @@ -138,15 +140,48 @@ class EncryptedContentAffinityCheck(CustomLogger): # If no encoded ID, check if encrypted_content itself is wrapped encrypted_content = item.get("encrypted_content") if encrypted_content and isinstance(encrypted_content, str): - ( - model_id, - _, - ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content) + model_id = EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(encrypted_content) if model_id: return model_id return None + @staticmethod + def _anthropic_content_blocks(messages: object) -> Iterator[Mapping[str, object]]: + if not isinstance(messages, list): + return iter(()) + return ( + cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + for message in cast(list[object], messages) # cast-ok: narrowed by isinstance + if isinstance(message, Mapping) + for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance + if isinstance(content, list) + for block in cast(list[object], content) # cast-ok: narrowed by isinstance + if isinstance(block, Mapping) + ) + + @staticmethod + def _model_id_from_wrapped_encrypted_content(encrypted_content: str) -> str | None: + model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content) + return model_id or None + + @staticmethod + def _extract_model_id_from_anthropic_messages(messages: object) -> str | None: + return next( + ( + model_id + for block in EncryptedContentAffinityCheck._anthropic_content_blocks(messages) + if (encrypted_content := encrypted_content_of_block(block)) is not None + if ( + model_id := EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content( + encrypted_content + ) + ) + is not None + ), + None, + ) + @staticmethod def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None: for deployment in healthy_deployments: @@ -248,13 +283,14 @@ class EncryptedContentAffinityCheck(CustomLogger): if "litellm_metadata" in request_kwargs: request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True - request_input: Final = request_kwargs.get("input") - model_id: Final = self._extract_model_id_from_input(request_input) + model_id: Final = self._extract_model_id_from_input( + request_kwargs.get("input") + ) or self._extract_model_id_from_anthropic_messages(request_kwargs.get("messages")) if not model_id: return typed_healthy_deployments verbose_router_logger.debug( - "EncryptedContentAffinityCheck: decoded model_id=%s from input item IDs", + "EncryptedContentAffinityCheck: decoded model_id=%s from the request's encrypted content markers", model_id, ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 2cbfaa17d23..2aa84f05a86 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -77,6 +77,16 @@ def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_reje assert "reasoning" in responses_kwargs +def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + extra_kwargs={"custom_llm_provider": "openai", "include": ["file_search_call.results"]}, + ) + assert responses_kwargs["include"] == ["reasoning.encrypted_content", "file_search_call.results"] + + def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key(): responses_kwargs = _build_responses_kwargs( max_tokens=1024, diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index dac991a41c4..e858f7eb0a8 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1656,3 +1656,43 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen assert request_kwargs.get("_encrypted_content_affinity_pinned") is True finally: router.discard() + + +@pytest.mark.asyncio +async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_through_the_bridge(): + """ + Claude Code behind /v1/messages replays the encrypted reasoning the bridge packed + into a thinking block's signature (or a redacted block's data). The pin has to be + read from those blocks because the bridge builds the Responses `input` only after + the router has picked a deployment. + """ + check = EncryptedContentAffinityCheck() + deployments = [ + {"model_info": {"id": "openai-org-a"}, "litellm_params": {"model": "openai/gpt-5.1"}}, + {"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5.1"}}, + ] + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", "openai-org-b") + request_kwargs = { + "messages": [ + {"role": "user", "content": "Solve the zebra puzzle"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, + {"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"}, + {"type": "text", "text": "The zebra owner lives in the green house."}, + ], + }, + {"role": "user", "content": "And who drinks water?"}, + ], + } + + pinned = await check.async_filter_deployments( + model="gpt-5.1", + healthy_deployments=deployments, + messages=None, + request_kwargs=request_kwargs, + ) + + assert [d["model_info"]["id"] for d in pinned] == ["openai-org-b"] + assert request_kwargs["_encrypted_content_affinity_pinned"] is True From 7563d94e65591c2ed3b909c196a3eb76e61625fd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:29:55 -0700 Subject: [PATCH 252/411] test(responses): assert the streamed response.completed event echoes the named tool_choice --- .../test_streaming_iterator_transformation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 29061db97dc..70f0957890d 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -679,6 +679,9 @@ def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape() -> None: "response.in_progress", "response.completed", ] - assert response_events[0].response.tool_choice == {"type": "function", "name": "run_command"} - assert response_events[1].response.tool_choice == {"type": "function", "name": "run_command"} + assert [event.response.tool_choice for event in response_events] == [ + {"type": "function", "name": "run_command"}, + {"type": "function", "name": "run_command"}, + {"type": "function", "name": "run_command"}, + ] assert any(getattr(event, "type", None) == "response.output_item.done" for event in events) From a68fe4e4d9eb8b55efd33be65ef4e1f1f1932c62 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:31:48 -0700 Subject: [PATCH 253/411] fix(responses): keep the client's usage shape when the logging copy cannot re-validate the response --- litellm/responses/streaming_iterator.py | 41 ++++++++------ .../responses/test_streaming_iterator.py | 54 ++++++++++++++++--- 2 files changed, 72 insertions(+), 23 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 658a0e8ad5d..40ff88fc557 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload, runti import httpx from openai._streaming import SSEDecoder -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from typing_extensions import TypeIs import litellm @@ -439,18 +439,7 @@ class BaseResponsesAPIStreamingIterator: if self._persist_completed_response_before_logging: self._persist_completed_response_to_cache(is_async=is_async) - # Create a copy for logging to avoid modifying the response object that will be returned to the user - # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) - # to chat completion format (prompt_tokens/completion_tokens) for internal logging - # Use model_dump + model_validate instead of deepcopy to avoid pickle errors with - # Pydantic ValidatorIterator when response contains tool_choice with allowed_tools (fixes #17192) - logging_response = self.completed_response - if self.completed_response is not None and hasattr(self.completed_response, "model_dump"): - try: - logging_response = type(self.completed_response).model_validate(self.completed_response.model_dump()) - except Exception: - # Fallback to original if serialization fails - pass + logging_response: Final[object] = _logging_copy(self.completed_response) self._restore_provider_response_headers(logging_response) end_time: Final = datetime.now() @@ -489,10 +478,10 @@ class BaseResponsesAPIStreamingIterator: def _restore_provider_response_headers(self, logging_response: object) -> None: """Re-apply the provider's response headers to the copy handed to logging callbacks. - ``model_validate(model_dump())`` above drops pydantic private attributes, so the + ``model_validate(model_dump())`` in ``_logging_copy`` drops pydantic private attributes, so the ``_hidden_params`` the provider transform set on the nested response are lost. Returns early - when that copy fell back to the original event, so logging-only state never lands on the - object the caller is iterating. + when the event was not a pydantic model and logging got the original, so logging-only state + never lands on the object the caller is iterating. """ if logging_response is self.completed_response: return @@ -1294,6 +1283,26 @@ def _add_text_like_part_events( ) +def _logging_copy(event: object) -> object: + """Hand logging callbacks a copy, so their usage rewrite (Responses shape to chat shape) never + reaches the event the caller is iterating. The round trip through ``model_dump`` sidesteps the + deepcopy pickle errors of #17192; when a provider payload fails validation (LIT-7391), shallow + copies of the event and its nested response still keep the caller's ``usage`` attribute separate.""" + if not isinstance(event, BaseModel): + return event + try: + return type(event).model_validate(event.model_dump()) + except Exception: + return _detached_shallow_copy(event) + + +def _detached_shallow_copy(event: BaseModel) -> BaseModel: + nested: Final[object] = getattr(event, "response", None) + if isinstance(nested, BaseModel): + return event.model_copy(update={"response": nested.model_copy()}) + return event.model_copy() + + def _usage_as_model(usage: object) -> ResponseAPIUsage | None: if isinstance(usage, ResponseAPIUsage): return usage diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 60ce08666b6..5e0e794d93e 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -18,6 +18,7 @@ from litellm.responses.streaming_iterator import ( SyncResponsesAPIStreamingIterator, ) from litellm.types.llms.openai import ( + ResponseAPIUsage, ResponseCompletedEvent, ResponsesAPIResponse, ResponsesAPIStreamEvents, @@ -329,8 +330,6 @@ def test_run_post_success_hooks_does_not_report_generation_time_as_overhead(): def _responses_api_response_with_usage() -> ResponsesAPIResponse: - from litellm.types.llms.openai import ResponseAPIUsage - return ResponsesAPIResponse( id="resp_lit6427", created_at=int(datetime(2025, 1, 1).timestamp()), @@ -383,8 +382,6 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): from litellm.responses.streaming_iterator import _stamp_responses_usage_cost - from litellm.types.llms.openai import ResponseAPIUsage - response = _unvalidated_response_with_dict_usage( { "input_tokens": 29, @@ -406,8 +403,6 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): from litellm.responses.streaming_iterator import _stamp_responses_usage_cost - from litellm.types.llms.openai import ResponseAPIUsage - response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149}) logging_obj = Mock(spec=LiteLLMLoggingObj) logging_obj._response_cost_calculator.return_value = 0.000704 @@ -586,5 +581,50 @@ async def test_streaming_logging_copy_fallback_leaves_caller_event_untouched(): with patch.object(type(iterator.completed_response), "model_dump", side_effect=ValueError("cannot serialize")): iterator._log_completed_response(is_async=True) - assert logged == [iterator.completed_response] + assert len(logged) == 1 + assert logged[0] is not iterator.completed_response + assert logged[0].response is not iterator.completed_response.response + assert logged[0].response._hidden_params["headers"]["apim-request-id"] == "azure-correlation-1" assert iterator.completed_response.response._hidden_params == {} + + +def _unvalidated_completed_config() -> Mock: + """Config whose completed event carries a Perplexity-style response that fails validation + (``truncation: ""``) and already holds the stamped ``ResponseAPIUsage``.""" + mock_config = Mock(spec=BaseResponsesAPIConfig) + + def _transform(model, parsed_chunk, logging_obj): + response = _unvalidated_response_with_dict_usage( + ResponseAPIUsage(input_tokens=29, output_tokens=373, total_tokens=402, cost={"total_cost": 0.0001}) + ) + return ResponseCompletedEvent(type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=response) + + mock_config.transform_streaming_response.side_effect = _transform + return mock_config + + +@pytest.mark.asyncio +async def test_streaming_logging_copy_keeps_client_usage_when_response_fails_validation(): + """LIT-7391: the logging copy cannot round-trip a response that fails validation, and logging + rewrites the assembled response's usage to chat shape in place, so the event handed to logging + must never be the one the caller receives.""" + logging_obj = _logging_obj_stub() + logging_obj.stream = True + logged: list[object] = [] + logging_obj.dispatch_success_handlers = _capture_dispatch(logged) + logging_obj._on_deferred_stream_complete = None + + iterator = _make_header_iterator(headers={}, config=_unvalidated_completed_config(), logging_obj=logging_obj) + events = [event async for event in iterator] + + assert len(logged) == 1 + now = datetime.now() + LiteLLMLoggingObj._get_assembled_streaming_response( + logging_obj, logged[0], start_time=now, end_time=now, is_async=True, streaming_chunks=[] + ) + assert logged[0].response.usage["prompt_tokens"] == 29 + + client_usage = events[-1].response.usage + assert isinstance(client_usage, ResponseAPIUsage) + assert client_usage.input_tokens == 29 + assert client_usage.cost == pytest.approx(0.0001) From 969d152c4bdc781e706ffffcd496ab1f3332c444 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 9 Sep 2026 16:40:25 -0700 Subject: [PATCH 254/411] fix(jwt): evict jwt_key_mapping cache when a virtual key is deleted The FK cascade drops the LiteLLM_JWTKeyMapping row, but the cached jwt_key_mapping:{claim}:{value} entry still resolved to the deleted token hash, so every JWT call from that identity failed until virtual_key_mapping_cache_ttl expired instead of auto-registering against a recreated key. delete_verification_tokens now snapshots the mapping cache keys before the delete and evicts them across replicas afterwards, the same way /key/regenerate already does. Claude-Session: https://claude.ai/code/session_011Tn3657NkV6ojLqewL64Kb --- .../key_management_endpoints.py | 19 ++++ .../test_key_management_endpoints.py | 98 +++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index f46c4170071..749a940de0e 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4559,6 +4559,23 @@ async def delete_verification_tokens( litellm_changed_by=litellm_changed_by, ) + # Snapshot before the delete: the FK cascade drops the mapping rows, but their + # cached jwt_key_mapping entries still resolve to the now-dead token (LIT-5380). + jwt_mapping_cache_keys: Final[tuple[str, ...]] = tuple( + cache_key + for keys_for_token in await asyncio.gather( + *( + get_jwt_key_mapping_cache_keys_for_token( + hashed_token=key.token, + prisma_client=prisma_client, + ) + for key in authorized_keys + if key.token is not None + ) + ) + for cache_key in keys_for_token + ) + if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value: deleted_tokens = await prisma_client.delete_data(tokens=tokens) if deleted_tokens is not None and len(deleted_tokens) != len(tokens): @@ -4571,6 +4588,8 @@ async def delete_verification_tokens( if len(deleted_tokens) != len(tokens): failed_tokens = [token for token in tokens if token not in deleted_tokens] + await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) + else: raise Exception("DB not connected. prisma_client is None") except Exception as e: diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index d2aeba18f7d..65cc23ea67f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5085,6 +5085,104 @@ async def test_delete_verification_tokens_persists_deleted_keys(monkeypatch): assert len(deleted_keys) == 2 +class _JWTMappingRow: + def __init__(self, token, jwt_claim_name, jwt_claim_value): + self.token = token + self.jwt_claim_name = jwt_claim_name + self.jwt_claim_value = jwt_claim_value + + +class _CascadingJWTMappingTable: + """Mapping rows that LiteLLM_JWTKeyMapping_token_fkey drops when their key is deleted.""" + + def __init__(self, rows): + self.rows = rows + + async def find_many(self, where, **kwargs): + return [row for row in self.rows if row.token == where["token"]] + + def cascade(self, deleted_tokens): + self.rows = [row for row in self.rows if row.token not in deleted_tokens] + + +class _RecordingEvict: + def __init__(self): + self.cache_keys = () + + async def __call__(self, cache_keys, user_api_key_cache): + self.cache_keys = tuple(cache_keys) + + +@pytest.mark.asyncio +async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypatch): + """Deleting a key must evict its jwt_key_mapping cache entries (LIT-5380). + + The FK cascade removes the mapping rows, so a surviving cache entry would keep + resolving the deleted token hash and 401 every JWT call from that identity until + virtual_key_mapping_cache_ttl expires, instead of auto-registering again. + """ + jwt_table = _CascadingJWTMappingTable( + [_JWTMappingRow("hashed-token-1", "email", "user@example.com")] + ) + + key1 = LiteLLM_VerificationToken( + token="hashed-token-1", + user_id="user-123", + team_id=None, + key_alias="jwt-mapped-key", + spend=0.0, + max_budget=None, + models=[], + aliases={}, + config={}, + permissions={}, + metadata={}, + model_max_budget={}, + model_spend={}, + soft_budget_cooldown=False, + allowed_routes=[], + ) + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[key1] + ) + mock_prisma_client.db.litellm_jwtkeymapping = jwt_table + mock_prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + + async def cascading_delete_data(tokens): + jwt_table.cascade(tokens) + return list(tokens) + + mock_prisma_client.delete_data = AsyncMock(side_effect=cascading_delete_data) + + recording_evict = _RecordingEvict() + monkeypatch.setattr( + "litellm.proxy.management_endpoints.key_management_endpoints.evict_and_broadcast", + recording_evict, + ) + monkeypatch.setattr( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + lambda token: token, + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + mock_prisma_client, + ) + + await delete_verification_tokens( + tokens=["hashed-token-1"], + user_api_key_cache=MagicMock(), + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", + api_key="sk-admin", + user_role=LitellmUserRoles.PROXY_ADMIN.value, + ), + ) + + assert recording_evict.cache_keys == ("jwt_key_mapping:email:user@example.com",) + + @pytest.mark.asyncio async def test_delete_key_fn_persists_deleted_keys(monkeypatch): from litellm.proxy._types import KeyRequest From 2000642592670baf38c46f4c6e3bcb851c59d79e Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 16:43:50 -0700 Subject: [PATCH 255/411] fix(router): resolve route candidate ids through the router's own resolver (#40491) get_candidate_model_ids_for_route (added in #40280 for the encrypted-content affinity check) reconstructed the candidate pool by unioning the model_name and team indexes with pattern_router.route. That diverged from how the router actually resolves a route: it took a union instead of the first matching path, and pattern_router.route only matches the literal name, so a provider-qualified pattern (matched by get_deployments_by_pattern, which retries the {provider}/{model} form) was missed and the default deployment was ignored. For an affinity follow-up on a wildcard or team-public route, that mismatch could strip encrypted reasoning on a same-group cooldown, or return a 503 on a real cross-path switch. Delegate the non-model_name case to _try_early_resolve_deployments_for_model_not_in_names, the same resolver _common_checks_available_deployment uses, so candidate membership follows the router's real precedence. With include_team_models left off it stays read-only and does not raise. Behavior for concrete model groups and routing groups is unchanged. Claude-Session: https://claude.ai/code/session_01KAumQbhzk6jdWWHFLA8Jar Co-authored-by: Claude Opus 4.8 --- litellm/router.py | 35 +++++++++++++++++-------------- tests/test_litellm/test_router.py | 7 ++++++- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 88601e5b97b..f9d4bf1428e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11170,28 +11170,31 @@ class Router: def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: """ - Deployment ids that could serve ``model`` for ``team_id``, unioned across the paths - the router resolves a route through: ``model_group_alias``, a routing group, the - ``model_name`` and team indexes, and wildcard pattern routes. Read-only and - side-effect-free, unlike ``_common_checks_available_deployment`` which also applies - fallbacks and can raise. Lets a pre-call check tell a genuine cross-group route from - same-group unavailability without re-deriving that precedence at the call site, and - without leaking deployment ids into request kwargs bound for the provider. + Deployment ids that could serve ``model`` for ``team_id``, following the same + precedence ``_common_checks_available_deployment`` uses to build a candidate pool: + ``model_group_alias``, then a routing group, then the first matching early-resolve + path for a name that is not a ``model_name`` (team route, wildcard pattern via + ``get_deployments_by_pattern``, team pattern router, default deployment), then the + ``model_name`` and team indexes. Delegating to the router's own resolvers keeps this + aligned with how a route actually resolves rather than re-deriving it, and unlike + ``_common_checks_available_deployment`` it is read-only: it does not apply request + fallbacks and (with ``include_team_models`` left off) does not raise. Lets a pre-call + check tell a genuine cross-group route from same-group unavailability without leaking + deployment ids into request kwargs bound for the provider. """ resolved: Final = self._get_model_from_alias(model=model) or model routing_group_members: Final = self._get_routing_group_deployments(model=resolved, team_id=team_id) if routing_group_members is not None: return self._deployment_ids(routing_group_members) - if resolved in self.model_names: - return self._deployment_ids(self._get_all_deployments(model_name=resolved, team_id=team_id)) - team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None - return self._deployment_ids( - ( - *self._get_all_deployments(model_name=resolved, team_id=team_id), - *(self.pattern_router.route(resolved) or ()), - *((team_router.route(resolved) or ()) if team_router is not None else ()), - ) + early: Final = self._try_early_resolve_deployments_for_model_not_in_names( + model=resolved, request_team_id=team_id ) + if early is not None: + early_deployments: Final = early[1] + return self._deployment_ids( + (early_deployments,) if isinstance(early_deployments, Mapping) else early_deployments + ) + return self._deployment_ids(self._get_all_deployments(model_name=resolved, team_id=team_id)) @staticmethod def _deployment_ids(deployments: Sequence[Mapping[str, object]]) -> frozenset[str]: diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 1e0327e5ac2..f5b1f79ad73 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -15023,7 +15023,9 @@ def test_get_candidate_model_ids_for_route_covers_model_name_and_pattern(): pre-call check can tell a genuine cross-group route from same-group unavailability. A concrete model group returns its member ids; a wildcard/pattern deployment is included for a concrete model it matches, which the bare model_name index misses. - Regression guard for the LIT-7195 tier-change discriminator's team/pattern gaps. + The unprefixed-name case must resolve through get_deployments_by_pattern (which retries + the provider-qualified form), not a bare pattern_router.route that only sees the literal + name. Regression guard for the LIT-7195 tier-change discriminator's team/pattern gaps. """ router = Router( model_list=[ @@ -15047,6 +15049,9 @@ def test_get_candidate_model_ids_for_route_covers_model_name_and_pattern(): assert router.get_candidate_model_ids_for_route(model="grp") == frozenset({"dep-a", "dep-b"}) assert "dep-wild" in router.get_candidate_model_ids_for_route(model="openai/gpt-4o-some-new-model") + # unprefixed name whose provider resolves to openai: only get_deployments_by_pattern's + # provider-qualified retry matches "openai/*"; a bare route() on the literal name misses it + assert "dep-wild" in router.get_candidate_model_ids_for_route(model="gpt-5") def test_deployment_ids_stringifies_ids_and_skips_entries_without_a_model_info_id(): From 70c238dde0c4bef5cbb976fbd382320a8021352d Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:54:43 -0700 Subject: [PATCH 256/411] fix(ui): preserve dotted MCP tool argument names --- ...MCPToolArgumentsForm.integration.test.tsx} | 114 +++++++++++++++++- .../mcp_tools/MCPToolArgumentsForm.tsx | 44 ++++--- 2 files changed, 141 insertions(+), 17 deletions(-) rename ui/litellm-dashboard/src/components/mcp_tools/{MCPToolArgumentsForm.test.tsx => MCPToolArgumentsForm.integration.test.tsx} (61%) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx similarity index 61% rename from ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.test.tsx rename to ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx index d6e614da5eb..a4302f9fd6a 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { render, screen } from "@testing-library/react"; +import { fireEvent, render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, it, expect } from "vitest"; import MCPToolArgumentsForm, { MCPToolArgumentsFormRef } from "./MCPToolArgumentsForm"; @@ -26,6 +26,118 @@ const submitError = async (ref: React.RefObject) }; describe("MCPToolArgumentsForm", () => { + it("keeps dotted arguments separate from a same-prefix object and converts their values", async () => { + const ref = renderForm({ + type: "object", + properties: { + "filter.category": { type: "string" }, + filter: { type: "object" }, + "page.limit": { type: "integer" }, + query: { type: "string" }, + }, + required: ["filter.category"], + }); + + fireEvent.change(screen.getByRole("textbox", { name: "filter.category *" }), { + target: { value: "invoices" }, + }); + fireEvent.change(screen.getByRole("textbox", { name: "filter", exact: true }), { + target: { value: '{"category":"receipts","metadata":{"region":"eu"}}' }, + }); + fireEvent.change(screen.getByRole("spinbutton", { name: "page.limit" }), { target: { value: "7" } }); + fireEvent.change(screen.getByRole("textbox", { name: "query" }), { target: { value: "September" } }); + + const expected = { + "filter.category": "invoices", + filter: { category: "receipts", metadata: { region: "eu" } }, + "page.limit": 7, + query: "September", + }; + await expect(submit(ref)).resolves.toEqual(expected); + }); + + it("shows required validation on the literal dotted field and accepts a correction", async () => { + const ref = renderForm({ + type: "object", + properties: { "filter.category": { type: "string" } }, + required: ["filter.category"], + }); + + expect(await submitError(ref)).toEqual({ + errorFields: [{ name: ["filter.category"], errors: ["Please enter filter.category"] }], + }); + expect(await screen.findByText("Please enter filter.category")).toBeInTheDocument(); + expect(screen.getByRole("textbox", { name: "filter.category *" })).toHaveAttribute("aria-invalid", "true"); + + fireEvent.change(screen.getByRole("textbox", { name: "filter.category *" }), { + target: { value: "invoices" }, + }); + await expect(submit(ref)).resolves.toEqual({ "filter.category": "invoices" }); + }); + + it("validates JSON for dotted arguments inside params and preserves their literal names", async () => { + const ref = renderForm({ + type: "object", + properties: { + params: { + type: "object", + properties: { "filter.options": { type: "object" } }, + required: ["filter.options"], + }, + }, + required: [], + }); + const field = screen.getByRole("textbox", { name: "filter.options *" }); + fireEvent.change(field, { target: { value: "invalid" } }); + + expect(await submitError(ref)).toEqual({ + errorFields: [{ name: ["filter.options"], errors: ["Invalid JSON"] }], + }); + expect(await screen.findByText("Invalid JSON")).toBeInTheDocument(); + + fireEvent.change(field, { target: { value: '{"region":"eu"}' } }); + await expect(submit(ref)).resolves.toEqual({ params: { "filter.options": { region: "eu" } } }); + }); + + it("resets dotted defaults and positional values when the selected tool changes", async () => { + const ref = React.createRef(); + const { rerender } = render( + , + ); + expect(screen.getByRole("textbox", { name: "filter.category" })).toHaveValue("invoices"); + await expect(submit(ref)).resolves.toEqual({ "filter.category": "invoices" }); + fireEvent.change(screen.getByRole("textbox", { name: "filter.category" }), { + target: { value: "edited" }, + }); + await expect(submit(ref)).resolves.toEqual({ "filter.category": "edited" }); + + rerender( + , + ); + expect(screen.getByRole("textbox", { name: "filter.category" })).toHaveValue("receipts"); + await expect(submit(ref)).resolves.toEqual({ query: "new tool", "filter.category": "receipts" }); + }); + it("returns typed values for a string, integer, number and boolean field", async () => { const user = userEvent.setup(); const ref = renderForm({ diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx index ca8c5697e6e..4fff3e57fd9 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx @@ -9,7 +9,10 @@ import { Textarea } from "@/components/ui/textarea"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { MCPTool, InputSchema, InputSchemaProperty } from "./types"; -type ToolFormValues = Record; +type ToolFormValues = { args: unknown[] }; + +const argumentValues = (schema: InputSchema, values: ToolFormValues): Record => + Object.fromEntries(Object.keys(schema.properties ?? {}).map((key, index) => [key, values.args[index]])); const STRING_SCHEMA_MESSAGES: Readonly> = { input: "Please enter input for this tool" }; @@ -38,7 +41,7 @@ type FieldError = { type: string; message: string }; const collectErrors = ( actualSchema: InputSchema, requiredMessages: Readonly>, - values: ToolFormValues, + values: Record, ): Record => { const entries = Object.entries(actualSchema.properties ?? {}).flatMap<[string, FieldError]>(([key, prop]) => { const value = values[key]; @@ -57,8 +60,18 @@ const collectErrors = ( const buildResolver = (actualSchema: InputSchema, requiredMessages: Readonly> = {}): Resolver => (values) => { - const errors = collectErrors(actualSchema, requiredMessages, values); - return Object.keys(errors).length > 0 ? { values: {}, errors } : { values, errors: {} }; + const errors = collectErrors(actualSchema, requiredMessages, argumentValues(actualSchema, values)); + if (Object.keys(errors).length === 0) return { values, errors: {} }; + return { + values: {}, + errors: { + args: Object.fromEntries( + Object.keys(actualSchema.properties ?? {}).flatMap((key, index) => + Object.hasOwn(errors, key) ? [[index, errors[key]]] : [], + ), + ), + }, + }; }; const labelFor = (key: string, prop: InputSchemaProperty, required: boolean): React.ReactNode => ( @@ -238,10 +251,7 @@ const MCPToolArgumentsForm = forwardRef( - () => - Object.fromEntries( - Object.entries(actualSchema.properties ?? {}).map(([key, prop]) => [key, getInitialValueForField(prop)]), - ), + () => ({ args: Object.values(actualSchema.properties ?? {}).map(getInitialValueForField) }), [actualSchema], ); @@ -255,7 +265,7 @@ const MCPToolArgumentsForm = forwardRef ({ getSubmitValues: async () => { - const values = form.getValues(); + const values = argumentValues(actualSchema, form.getValues()); const errors = collectErrors(actualSchema, requiredMessages, values); if (Object.keys(errors).length > 0) { await form.trigger(); @@ -286,14 +296,16 @@ const MCPToolArgumentsForm = forwardRef Input * } > - {(field) => } + {(field) => ( + + )} @@ -318,13 +330,13 @@ const MCPToolArgumentsForm = forwardRef - {Object.entries(actualSchema.properties).map(([key, prop]) => { + {Object.entries(actualSchema.properties).map(([key, prop], index) => { const required = actualSchema.required?.includes(key) ?? false; return ( {(field) => { @@ -375,7 +387,7 @@ const MCPToolArgumentsForm = forwardRef ); @@ -385,7 +397,7 @@ const MCPToolArgumentsForm = forwardRef ); From fb9bb60e80baed40d80ef0cbd7c0482fbcd20e62 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:55:52 -0700 Subject: [PATCH 257/411] test(mcp): cover auth diagnostics through HTTP handler --- .../mcp_server/test_mcp_server.py | 162 ++++++++---------- 1 file changed, 76 insertions(+), 86 deletions(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 9b6eaba3177..12805e355f9 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -3,6 +3,7 @@ import contextvars import os from datetime import datetime, timedelta from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -1866,96 +1867,85 @@ async def test_streamable_http_session_manager_is_stateless(): @pytest.mark.asyncio -async def test_mcp_routing_initialize_to_stateful_no_session_to_stateless(): - """ - Test that routing correctly sends: - - initialize (no mcp-session-id) → stateful manager (so client gets mcp-session-id) - - tools/list (no mcp-session-id) → stateless manager (curl, Inspector) - """ - try: - from litellm.proxy._experimental.mcp_server.server import ( - handle_streamable_http_mcp, - session_manager_stateful, - session_manager_stateless, +@pytest.mark.parametrize("debug", (False, True)) +@pytest.mark.parametrize( + ("method", "request_body", "stateful"), + ( + ("POST", b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}', True), + ("POST", b'{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}', False), + ("GET", b"", False), + ("DELETE", b"", False), + ), +) +async def test_mcp_routing_initialize_to_stateful_no_session_to_stateless( + debug: bool, method: str, request_body: bytes, stateful: bool +) -> None: + from mcp.server.lowlevel.server import request_ctx + from mcp.shared.context import RequestContext + from starlette.requests import Request + from starlette.types import Message, Receive, Scope, Send + + from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy._experimental.mcp_server.mcp_debug import record_auth_resolution + from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution + + scope: Final[Scope] = {"type": "http", "method": method, "path": "/mcp", "headers": []} + receive: Final = AsyncMock(return_value={"type": "http.request", "body": request_body, "more_body": False}) + send: Final = AsyncMock() + observe_start: Final = AsyncMock() + body: Final[Message] = {"type": "http.response.body", "body": b"data: pong\n\n", "more_body": True} + + async def handle_request(request_scope: Scope, receive: Receive, outgoing: Send) -> None: + await outgoing({"type": "http.response.start", "status": 200, "headers": []}) + await observe_start(send.await_count) + context: Final = RequestContext( + request_id=1, meta=None, session=MagicMock(), lifespan_context=None, request=Request(request_scope) ) - except ImportError: - pytest.skip("MCP server not available") + token: Final = request_ctx.set(context) + try: + record_auth_resolution("s1", AuthResolution.stored_user_token) + finally: + request_ctx.reset(token) + await outgoing(body) - async def make_request(method_body: bytes, path: str = "/mcp/progress_test"): - scope = { - "type": "http", - "method": "POST", - "path": path, - "headers": [ - (b"content-type", b"application/json"), - (b"authorization", b"Bearer test-key"), - ], - } - receive = AsyncMock( - return_value={ - "type": "http.request", - "body": method_body, - "more_body": False, - } - ) - send = AsyncMock() - - stateless_called = [] - stateful_called = [] - - async def stateless_handle(s, r, se): - stateless_called.append(1) - - async def stateful_handle(s, r, se): - stateful_called.append(1) - - with ( - patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", - new_callable=AsyncMock, - return_value=(MagicMock(), None, ["progress_test"], None, None, None), + stateless_handle: Final = AsyncMock(side_effect=handle_request) + stateful_handle: Final = AsyncMock(side_effect=handle_request) + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=( + UserAPIKeyAuth(user_id="debug-user"), + None, + None, + None, + None, + {"x-litellm-mcp-debug": "true"} if debug else {}, ), - patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", - ), - patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", - True, - ), - patch.object( - session_manager_stateless, - "handle_request", - side_effect=stateless_handle, - ), - patch.object( - session_manager_stateful, - "handle_request", - side_effect=stateful_handle, - ), - patch.object( - session_manager_stateless, - "_server_instances", - {}, - ), - patch.object( - session_manager_stateful, - "_server_instances", - {}, - ), - ): - await handle_streamable_http_mcp(scope, receive, send) + ), + patch("litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True), + patch( + "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", + SimpleNamespace(handle_request=stateless_handle), + ), + patch( + "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", + SimpleNamespace(handle_request=stateful_handle), + ), + ): + await mcp_module.handle_streamable_http_mcp(scope, receive, send) - return bool(stateless_called), bool(stateful_called) - - # initialize → stateful - init_body = b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05"}}' - stateless_called, stateful_called = await make_request(init_body) - assert stateful_called and not stateless_called, "initialize (no session) should route to stateful, not stateless" - - # tools/list → stateless - tools_body = b'{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' - stateless_called, stateful_called = await make_request(tools_body) - assert stateless_called and not stateful_called, "tools/list (no session) should route to stateless, not stateful" + assert stateful_handle.await_count == (1 if stateful else 0) + assert stateless_handle.await_count == (0 if stateful else 1) + observe_start.assert_awaited_once_with(0 if debug and method == "POST" else 1) + assert send.await_count == 2 + assert send.call_args_list[0].args[0]["status"] == 200 + assert send.call_args_list[1].args[0] == body + headers: Final = dict(send.call_args_list[0].args[0]["headers"]) + if debug: + assert headers[b"x-mcp-debug-auth-resolution"] == (b"stored-user-token" if method == "POST" else b"unresolved") + else: + assert not any(name.startswith(b"x-mcp-debug") for name in headers) @pytest.mark.asyncio From 0b21b99ebd193872c520996cec79defad07ee108 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:57:15 -0700 Subject: [PATCH 258/411] fix(ui): type MCP argument resolver results explicitly --- .../mcp_tools/MCPToolArgumentsForm.integration.test.tsx | 2 +- .../src/components/mcp_tools/MCPToolArgumentsForm.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx index a4302f9fd6a..b1174d1d37d 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.integration.test.tsx @@ -41,7 +41,7 @@ describe("MCPToolArgumentsForm", () => { fireEvent.change(screen.getByRole("textbox", { name: "filter.category *" }), { target: { value: "invoices" }, }); - fireEvent.change(screen.getByRole("textbox", { name: "filter", exact: true }), { + fireEvent.change(screen.getByRole("textbox", { name: "filter" }), { target: { value: '{"category":"receipts","metadata":{"region":"eu"}}' }, }); fireEvent.change(screen.getByRole("spinbutton", { name: "page.limit" }), { target: { value: "7" } }); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx index 4fff3e57fd9..ab3213d9b37 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx @@ -1,6 +1,6 @@ import React, { forwardRef, useImperativeHandle, useMemo } from "react"; import { CircleHelp } from "lucide-react"; -import { useForm, type Resolver } from "react-hook-form"; +import { useForm, type Resolver, type ResolverResult } from "react-hook-form"; import { FieldGroup } from "@/components/ui/field"; import { FormField } from "@/components/shared/form/FormField"; import { Input } from "@/components/ui/input"; @@ -59,7 +59,7 @@ const collectErrors = ( const buildResolver = (actualSchema: InputSchema, requiredMessages: Readonly> = {}): Resolver => - (values) => { + (values): ResolverResult => { const errors = collectErrors(actualSchema, requiredMessages, argumentValues(actualSchema, values)); if (Object.keys(errors).length === 0) return { values, errors: {} }; return { From 6b721de3e530bc97758d59621e57e3a75a9ebadf Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 17:15:48 -0700 Subject: [PATCH 259/411] fix(databricks): route Unity model services through AI Gateway (#40492) Co-authored-by: Claude Code --- .../llms/databricks/chat/transformation.py | 6 +- litellm/llms/databricks/common_utils.py | 20 ++++-- .../test_databricks_chat_transformation.py | 13 ++++ .../test_databricks_partner_integration.py | 71 +++++++++++++++++++ 4 files changed, 101 insertions(+), 9 deletions(-) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 54c7040daf9..09aaf970dc5 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -250,8 +250,10 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - api_base = self._get_api_base(api_base) - complete_url: Final = f"{api_base}/chat/completions" + use_ai_gateway: Final = model.removeprefix("databricks/").count(".") >= 2 + api_base = self._get_api_base(api_base, use_ai_gateway=use_ai_gateway) + url_base: Final = api_base.rstrip("/") if use_ai_gateway else api_base + complete_url: Final = f"{url_base}/chat/completions" return complete_url def get_supported_openai_params(self, model: str | None = None) -> list: diff --git a/litellm/llms/databricks/common_utils.py b/litellm/llms/databricks/common_utils.py index 7695b1cb35e..a4ec2c5378b 100644 --- a/litellm/llms/databricks/common_utils.py +++ b/litellm/llms/databricks/common_utils.py @@ -177,19 +177,13 @@ class DatabricksBase: # Default: just litellm return f"litellm/{version}" - def _get_api_base(self, api_base: str | None) -> str: - """ - Get the Databricks API base URL. - - If not provided, attempts to get it from the Databricks SDK. - """ + def _get_api_base(self, api_base: str | None, use_ai_gateway: bool = False) -> str: if api_base is None: try: from databricks.sdk import WorkspaceClient databricks_client: Final = WorkspaceClient() api_base = f"{databricks_client.config.host}/serving-endpoints" - return api_base except ImportError: raise DatabricksException( status_code=400, @@ -198,6 +192,18 @@ class DatabricksBase: "or install the databricks-sdk Python library." ), ) + + if not use_ai_gateway: + return api_base + + normalized_api_base: Final = api_base.rstrip("/") + if normalized_api_base.endswith("/ai-gateway/mlflow/v1"): + return normalized_api_base + if normalized_api_base.endswith("/serving-endpoints"): + return f"{normalized_api_base.removesuffix('/serving-endpoints')}/ai-gateway/mlflow/v1" + api_base_parts: Final = urlsplit(normalized_api_base) + if api_base_parts.path in ("", "/"): + return f"{normalized_api_base}/ai-gateway/mlflow/v1" return api_base def _get_oauth_m2m_token( diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 38d5844b9b9..caf7bed7385 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -255,6 +255,19 @@ def test_transform_messages_sanitizes_empty_content(): assert result[1]["content"] == "Hi" +def test_transform_request_preserves_unity_model_service_name(): + config = DatabricksConfig() + result = config.transform_request( + model="system.ai.kimi-k3", + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["model"] == "system.ai.kimi-k3" + + def test_transform_request_strips_thinking_blocks_and_reasoning_content(): """Regression for LIT-6762: replaying an assistant turn that litellm decorated with `thinking_blocks` / `reasoning_content` made Databricks 400 with diff --git a/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py b/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py index 39198bb20f3..c6ad78366f7 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py +++ b/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py @@ -657,6 +657,77 @@ class TestEndpointURLConstruction: assert api_base.endswith("/chat/completions") + def test_chat_gateway_endpoint_for_unity_model_on_legacy_base(self, monkeypatch): + from litellm.llms.databricks.chat.transformation import DatabricksConfig + + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + + url = DatabricksConfig().get_complete_url( + api_base="https://test.net/serving-endpoints", + api_key="test-key", + model="system.ai.kimi-k3", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://test.net/ai-gateway/mlflow/v1/chat/completions" + + def test_chat_gateway_endpoint_preserves_explicit_gateway_base(self, monkeypatch): + from litellm.llms.databricks.chat.transformation import DatabricksConfig + + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + + url = DatabricksConfig().get_complete_url( + api_base="https://test.net/ai-gateway/mlflow/v1/", + api_key="test-key", + model="system.ai.kimi-k3", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://test.net/ai-gateway/mlflow/v1/chat/completions" + + def test_chat_gateway_preserves_unity_model_service_name_with_explicit_base(self, monkeypatch): + from litellm.llms.databricks.chat.transformation import DatabricksConfig + + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + config = DatabricksConfig() + request = config.transform_request( + model="catalog.schema.kimi-k3", + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert config.get_complete_url( + api_base="https://test.net/ai-gateway/mlflow/v1", + api_key="test-key", + model="catalog.schema.kimi-k3", + optional_params={}, + litellm_params={}, + ) == "https://test.net/ai-gateway/mlflow/v1/chat/completions" + assert request["model"] == "catalog.schema.kimi-k3" + + def test_chat_legacy_endpoint_remains_default(self, monkeypatch): + from litellm.llms.databricks.chat.transformation import DatabricksConfig + + monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) + monkeypatch.delenv("DATABRICKS_CLIENT_SECRET", raising=False) + + url = DatabricksConfig().get_complete_url( + api_base="https://test.net/serving-endpoints", + api_key="test-key", + model="databricks-kimi-k3", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://test.net/serving-endpoints/chat/completions" + def test_embeddings_endpoint(self, monkeypatch): """Embeddings endpoint is correctly appended.""" monkeypatch.delenv("DATABRICKS_CLIENT_ID", raising=False) From 892d20d86f1470c302b116c3a6048ce5a9dd30f8 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:34:24 -0700 Subject: [PATCH 260/411] fix(mcp): refresh tools when editing server connection settings --- .../_components/editToolPreview.test.ts | 87 ++++++++++++++ .../_components/editToolPreview.ts | 53 +++++++++ .../mcp_server_edit.integration.test.tsx | 111 +++++++++++++++++- .../_components/mcp_server_edit.tsx | 47 ++++++-- .../_components/mcp_tool_configuration.tsx | 2 +- 5 files changed, 285 insertions(+), 15 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.test.ts new file mode 100644 index 00000000000..9dcfc1a470e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { getEditToolPreview } from "./editToolPreview"; + +const saved = { + url: "https://example.com/mcp", + transport: "http", + auth_type: "basic", + static_headers: [{ header: "X-Tenant", value: "original" }], +}; + +describe("getEditToolPreview", () => { + it("keeps saved discovery for unchanged settings and unrelated edits", () => { + expect(getEditToolPreview({ ...saved, server_name: "renamed" }, saved)).toEqual({ kind: "saved" }); + }); + + it("previews URL changes with the existing server credential left for server-side inheritance", () => { + expect(getEditToolPreview({ ...saved, url: "https://correct.example/mcp" }, saved)).toEqual({ + kind: "preview", + config: { + url: "https://correct.example/mcp", + transport: "http", + auth_type: "basic", + static_headers: { "X-Tenant": "original" }, + credentials: undefined, + }, + }); + }); + + it("uses edited static headers and only the static auth value", () => { + expect( + getEditToolPreview( + { + ...saved, + static_headers: [{ header: "X-Tenant", value: "corrected" }], + credentials: { auth_value: "user:password", access_token: "old-oauth-token", client_secret: "old-client" }, + }, + saved, + ), + ).toEqual({ + kind: "preview", + config: { + url: saved.url, + transport: "http", + auth_type: "basic", + static_headers: { "X-Tenant": "corrected" }, + credentials: { auth_value: "user:password" }, + }, + }); + }); + + it.each(["", "https://", "file:///tmp/server"])("does not connect to an incomplete or unsupported URL: %s", (url) => { + expect(getEditToolPreview({ ...saved, url }, saved)).toEqual({ kind: "incomplete" }); + }); + + it("waits for a static header value before connecting", () => { + const values = { ...saved, static_headers: [{ header: "X-Tenant", value: "" }] }; + expect(getEditToolPreview(values, saved)).toEqual({ kind: "incomplete" }); + }); + + it("waits for credentials when switching from None to Basic Auth", () => { + expect(getEditToolPreview(saved, { ...saved, auth_type: "none" })).toEqual({ kind: "incomplete" }); + }); + + it("does not forward old credentials when switching to None", () => { + const result = getEditToolPreview( + { ...saved, auth_type: "none", credentials: { auth_value: "old-secret" } }, + saved, + ); + expect(result.kind).toBe("preview"); + if (result.kind === "preview") expect(result.config.credentials).toBeUndefined(); + }); + + it.each(["oauth2", "true_passthrough", "oauth_delegate", "oauth2_token_exchange", "oauth2_id_jag", "aws_sigv4"])( + "preserves the existing discovery path for %s", + (auth_type) => { + expect(getEditToolPreview({ ...saved, auth_type, url: "https://changed.example/mcp" }, saved)).toEqual({ + kind: "saved", + }); + }, + ); + + it("keeps stdio and OpenAPI on their existing discovery path", () => { + for (const transport of ["stdio", "openapi"]) { + expect(getEditToolPreview({ ...saved, transport }, saved)).toEqual({ kind: "saved" }); + } + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.ts new file mode 100644 index 00000000000..eb46c33c4e7 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.ts @@ -0,0 +1,53 @@ +import { AUTH_TYPE, TRANSPORT } from "@/components/mcp_tools/types"; +import { AUTH_TYPES_REQUIRING_AUTH_VALUE, reduceStaticHeaders } from "./createServerPayload"; + +const connectionConfig = (values: Readonly>) => { + const credentials = values.credentials; + const authValue = + credentials && typeof credentials === "object" && "auth_value" in credentials ? credentials.auth_value : undefined; + const needsAuthValue = + typeof values.auth_type === "string" && AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(values.auth_type); + return { + url: typeof values.url === "string" ? values.url : "", + transport: typeof values.transport === "string" ? values.transport : "", + auth_type: typeof values.auth_type === "string" ? values.auth_type : "", + static_headers: Object.fromEntries( + Object.entries(reduceStaticHeaders(values.static_headers)).sort(([a], [b]) => a.localeCompare(b)), + ), + credentials: + needsAuthValue && typeof authValue === "string" && authValue.trim() ? { auth_value: authValue } : undefined, + }; +}; + +type EditToolPreview = + | { readonly kind: "saved" } + | { readonly kind: "incomplete" } + | { readonly kind: "preview"; readonly config: ReturnType }; + +export const getEditToolPreview = ( + values: Readonly>, + initialValues: Readonly>, +): EditToolPreview => { + const staticAuth = + values.auth_type === AUTH_TYPE.NONE || + (typeof values.auth_type === "string" && AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(values.auth_type)); + if (!staticAuth || ![TRANSPORT.HTTP, TRANSPORT.SSE].includes(String(values.transport))) { + return { kind: "saved" }; + } + + const config = connectionConfig(values); + if (JSON.stringify(config) === JSON.stringify(connectionConfig(initialValues))) { + return { kind: "saved" }; + } + + const missingNewCredential = + config.auth_type !== initialValues.auth_type && + AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(config.auth_type) && + config.credentials === undefined; + const validUrl = URL.canParse(config.url) && ["http:", "https:"].includes(new URL(config.url).protocol); + const incompleteHeaders = Object.values(config.static_headers).some((value) => !value.trim()); + if (!validUrl || missingNewCredential || incompleteHeaders) { + return { kind: "incomplete" }; + } + return { kind: "preview", config }; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx index a9664191f3f..f3cd37cc580 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx @@ -1,6 +1,9 @@ import React from "react"; import { describe, it, expect, vi, beforeEach } from "vitest"; -import { render, screen, waitFor, act } from "@testing-library/react"; +import { render, screen, waitFor, act, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; + +import { selectOption } from "./testUtils"; import MCPServerEdit from "./mcp_server_edit"; import * as networking from "@/components/networking"; @@ -27,10 +30,6 @@ vi.mock("./mcp_server_cost_config", () => ({ default: () =>
, })); -vi.mock("./mcp_tool_configuration", () => ({ - default: () =>
, -})); - const BASE: MCPServer = { server_id: "srv_1", server_name: "srv", @@ -369,3 +368,105 @@ describe("mcp_server_edit save payload contract", () => { } }); }); + +describe("MCPServerEdit live tool preview", () => { + beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(networking.listMCPTools).mockResolvedValue({ + tools: [], + error: "connection_error", + message: "Saved credentials rejected", + }); + vi.mocked(networking.testMCPToolsListRequest).mockResolvedValue({ + tools: [ + { name: "echo", description: "Echo the supplied message", inputSchema: { type: "object", properties: {} } }, + ], + }); + }); + + const renderEditor = (server: MCPServer = BASE) => + render( + , + ); + + it("replaces the saved connection failure with tools after correcting Basic Auth without saving", async () => { + renderEditor(); + expect(await screen.findByText("Saved credentials rejected")).toBeInTheDocument(); + await selectOption("Authentication", "Basic Auth"); + fireEvent.change(screen.getByLabelText("Authentication Value"), { target: { value: "preview:correct" } }); + expect(screen.queryByText("Saved credentials rejected")).not.toBeInTheDocument(); + expect(screen.getByText("Loading tools...")).toBeInTheDocument(); + expect(networking.testMCPToolsListRequest).not.toHaveBeenCalled(); + fireEvent.click(await screen.findByRole("button", { name: "Flat List" })); + expect(screen.getByText("echo")).toBeInTheDocument(); + const expectedConfig = { + server_id: BASE.server_id, + url: BASE.url, + auth_type: "basic", + credentials: { auth_value: "preview:correct" }, + }; + expect(networking.testMCPToolsListRequest).toHaveBeenCalledExactlyOnceWith( + "access-token", + expect.objectContaining(expectedConfig), + ); + expect(networking.updateMCPServer).not.toHaveBeenCalled(); + }); + + it("refreshes tools when a static header is corrected", async () => { + renderEditor({ ...BASE, static_headers: { "X-Preview-Key": "wrong" } }); + expect(await screen.findByText("Saved credentials rejected")).toBeInTheDocument(); + fireEvent.change(screen.getByPlaceholderText("Header value"), { target: { value: "correct" } }); + fireEvent.click(await screen.findByRole("button", { name: "Flat List" })); + expect(screen.getByText("echo")).toBeInTheDocument(); + expect(networking.testMCPToolsListRequest).toHaveBeenCalledExactlyOnceWith( + "access-token", + expect.objectContaining({ static_headers: { "X-Preview-Key": "correct" } }), + ); + }); + + it("coalesces URL edits and ignores an older failed preview after the latest preview succeeds", async () => { + const user = userEvent.setup(); + const older = Promise.withResolvers<{ tools: never[]; error: string; message: string }>(); + vi.mocked(networking.testMCPToolsListRequest).mockImplementationOnce(() => older.promise); + renderEditor(); + expect(await screen.findByText("Saved credentials rejected")).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("MCP Server URL"), { target: { value: "https://first.example/mcp" } }); + await waitFor(() => expect(networking.testMCPToolsListRequest).toHaveBeenCalledTimes(1)); + await user.clear(screen.getByLabelText("MCP Server URL")); + await user.type(screen.getByLabelText("MCP Server URL"), "https://latest.example/mcp"); + expect(networking.testMCPToolsListRequest).toHaveBeenCalledTimes(1); + fireEvent.click(await screen.findByRole("button", { name: "Flat List" })); + expect(screen.getByText("echo")).toBeInTheDocument(); + expect(networking.testMCPToolsListRequest).toHaveBeenCalledTimes(2); + expect(networking.testMCPToolsListRequest).toHaveBeenLastCalledWith( + "access-token", + expect.objectContaining({ url: "https://latest.example/mcp" }), + ); + await act(async () => older.resolve({ tools: [], error: "connection_error", message: "Older request failed" })); + expect(screen.getByText("echo")).toBeInTheDocument(); + expect(screen.queryByText("Older request failed")).not.toBeInTheDocument(); + }); + + it("ignores a saved-record response after editing and restores saved discovery when changes are reverted", async () => { + const saved = Promise.withResolvers<{ tools: never[]; error: string; message: string }>(); + vi.mocked(networking.listMCPTools).mockImplementationOnce(() => saved.promise); + renderEditor(); + await waitFor(() => expect(networking.listMCPTools).toHaveBeenCalledTimes(1)); + fireEvent.change(screen.getByLabelText("MCP Server URL"), { target: { value: "https://correct.example/mcp" } }); + fireEvent.click(await screen.findByRole("button", { name: "Flat List" })); + expect(screen.getByText("echo")).toBeInTheDocument(); + await act(async () => saved.resolve({ tools: [], error: "connection_error", message: "Stale saved response" })); + expect(screen.getByText("echo")).toBeInTheDocument(); + expect(screen.queryByText("Stale saved response")).not.toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("MCP Server URL"), { target: { value: BASE.url } }); + expect(await screen.findByText("Saved credentials rejected")).toBeInTheDocument(); + expect(networking.listMCPTools).toHaveBeenCalledTimes(2); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx index 8793c45371a..5316a22ce27 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx @@ -52,6 +52,7 @@ import EnvVarsSection from "./EnvVarsSection"; import { validateMCPServerUrl, validateMCPServerName, normalizeToolOverrideMap } from "./utils"; import { EditServerFormValues, buildEditServerPayload, editPayloadErrorMessage } from "./editServerPayload"; import { toast } from "@/lib/toast"; +import { getEditToolPreview } from "./editToolPreview"; import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow"; import { MountedFormField, @@ -449,14 +450,27 @@ const MCPServerEdit: React.FC = ({ } }, [mcpServer]); - // Fetch tools when component mounts for a saved server + const toolPreview = getEditToolPreview(allFieldsValue(form), initialValues); + const toolPreviewKey = JSON.stringify(toolPreview); + useEffect(() => { - if (!mcpServer.server_id || mcpServer.server_id.trim() === "") { + let active = true; + setTools([]); + setToolsError(null); + setIsLoadingTools(false); + if (!accessToken || !mcpServer.server_id) return; + if (toolPreview.kind === "incomplete") { + setToolsError("Complete the URL, authentication, and header settings to load tools."); return; } - fetchTools(); + setIsLoadingTools(true); + const timer = setTimeout(() => fetchTools(() => active), toolPreview.kind === "preview" ? 500 : 0); + return () => { + active = false; + clearTimeout(timer); + }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [mcpServer, accessToken, userID, oauthTokenResponse?.access_token]); + }, [mcpServer, accessToken, userID, oauthTokenResponse?.access_token, toolPreviewKey]); // Invalidate a token authorized in this edit session once any mint-relevant field diverges from the // identity it was minted against (url, auth_type, oauth_flow_type, client creds/scopes, or the @@ -519,6 +533,7 @@ const MCPServerEdit: React.FC = ({ const previewWithStagedInteractiveToken = async ( isPassthrough: boolean, isBrowserHeldTokenMode: boolean, + isCurrent: () => boolean, ): Promise => { const stagedToken = !isPassthrough && !isBrowserHeldTokenMode && getEffectiveAuthType() === AUTH_TYPE.OAUTH2 @@ -550,6 +565,7 @@ const MCPServerEdit: React.FC = ({ registration_url: values.registration_url, }; const toolsResponse = await testMCPToolsListRequest(accessToken, previewConfig, stagedToken); + if (!isCurrent()) return true; if (toolsResponse.tools && !toolsResponse.error) { setTools(toolsResponse.tools); } else { @@ -557,15 +573,16 @@ const MCPServerEdit: React.FC = ({ setToolsError(toolsResponse.message || "Failed to load tools"); } } catch (error) { + if (!isCurrent()) return true; setTools([]); setToolsError(error instanceof Error ? error.message : "Failed to load tools"); } finally { - setIsLoadingTools(false); + if (isCurrent()) setIsLoadingTools(false); } return true; }; - const fetchTools = async () => { + const fetchTools = async (isCurrent: () => boolean) => { if (!accessToken || !mcpServer.server_id) return; // OBO/M2M/static auth is attached server-side from the stored credential, so @@ -574,6 +591,7 @@ const MCPServerEdit: React.FC = ({ // same way the Tools playground does. let customHeaders: Record | undefined; const isPassthrough = + toolPreview.kind === "saved" && getMcpOAuthMode({ auth_type: mcpServer.auth_type, oauth2_flow: mcpServer.oauth2_flow, @@ -581,9 +599,10 @@ const MCPServerEdit: React.FC = ({ }) === "passthrough"; const isBrowserHeldTokenMode = isClientForwardedTokenMode(getEffectiveAuthType()); - if (await previewWithStagedInteractiveToken(isPassthrough, isBrowserHeldTokenMode)) { + if (await previewWithStagedInteractiveToken(isPassthrough, isBrowserHeldTokenMode, isCurrent)) { return; } + if (!isCurrent()) return; if (isPassthrough || isBrowserHeldTokenMode) { const token = oauthTokenResponse?.access_token ?? @@ -591,6 +610,7 @@ const MCPServerEdit: React.FC = ({ ? getToken(mcpServer.server_id, userID)?.access_token ?? null : null); if (!token) { + setIsLoadingTools(false); setTools([]); setToolsError( isBrowserHeldTokenMode @@ -608,7 +628,15 @@ const MCPServerEdit: React.FC = ({ try { // include_disabled_tools: configuring the allowlist needs the full server // catalog, so tools toggled off still render (as unchecked) instead of vanishing. - const toolsResponse = await listMCPTools(accessToken, mcpServer.server_id, customHeaders, true); + const toolsResponse = + toolPreview.kind === "preview" + ? await testMCPToolsListRequest(accessToken, { + ...toolPreview.config, + server_id: mcpServer.server_id, + server_name: mcpServer.server_name || mcpServer.alias, + }) + : await listMCPTools(accessToken, mcpServer.server_id, customHeaders, true); + if (!isCurrent()) return; if (toolsResponse.tools && !toolsResponse.error) { setTools(toolsResponse.tools); @@ -617,10 +645,11 @@ const MCPServerEdit: React.FC = ({ setToolsError(toolsResponse.message || "Failed to load tools"); } } catch (error) { + if (!isCurrent()) return; setTools([]); setToolsError(error instanceof Error ? error.message : "Failed to load tools"); } finally { - setIsLoadingTools(false); + if (isCurrent()) setIsLoadingTools(false); } }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_tool_configuration.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_tool_configuration.tsx index 274bdf63e32..a3d62de9941 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_tool_configuration.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_tool_configuration.tsx @@ -433,7 +433,7 @@ const MCPToolConfiguration: React.FC = ({ {isLoadingTools && (
-

Loading tools from spec...

+

Loading tools...

)} From b8d7f68aeba2bd4a599895c910df085c2eb530a1 Mon Sep 17 00:00:00 2001 From: kerry-berri Date: Wed, 9 Sep 2026 17:46:12 -0700 Subject: [PATCH 261/411] Apply suggestion from @greptile-apps[bot] Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../src/components/llm_calls/chat_completion.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx index b813a35f687..c12a2c3cfb4 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx @@ -245,7 +245,7 @@ export async function makeOpenAIChatCompletionRequest( // Extract cost from usage object if available if (chunkWithUsage.usage.cost !== undefined && chunkWithUsage.usage.cost !== null) { - const parsedCost = parseFloat(chunkWithUsage.usage.cost); + const parsedCost = Number(chunkWithUsage.usage.cost); if (Number.isFinite(parsedCost)) { usageData.cost = parsedCost; } From 3258b732c74a344863e0445bd9e9bd99c248efc8 Mon Sep 17 00:00:00 2001 From: mrinal Date: Thu, 10 Sep 2026 00:46:54 +0000 Subject: [PATCH 262/411] chore(ui): bump smol-toml to fix GHSA-7w5x-hrqm-74c2 osv-scan failure Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ui/litellm-dashboard/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 44360e94392..1f459bc50ea 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -11402,9 +11402,9 @@ } }, "node_modules/smol-toml": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", - "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz", + "integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==", "dev": true, "license": "BSD-3-Clause", "engines": { From 245369764e80d113e80ec8269846e5257692b8bc Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:49:52 -0700 Subject: [PATCH 263/411] fix(mcp): preserve edited settings in static connection previews --- .../mcp_server/rest_endpoints.py | 11 +++- .../mcp_server/test_rest_endpoints.py | 66 +++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 5b74caee3a2..a72465f16f4 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1234,7 +1234,7 @@ if MCP_AVAILABLE: return client_id, client_secret, scopes _STAGED_AUTH_VALUE_AUTH_TYPES: Final = frozenset( - (MCPAuth.api_key, MCPAuth.bearer_token, MCPAuth.basic, MCPAuth.authorization) + (MCPAuth.api_key, MCPAuth.bearer_token, MCPAuth.basic, MCPAuth.authorization, MCPAuth.token) ) @dataclass(frozen=True, slots=True) @@ -1318,8 +1318,15 @@ if MCP_AVAILABLE: if _oauth2_flow == "client_credentials" and not request.token_url: _oauth2_flow = None + # Static previews inherit credentials before this step, but must not resolve back to + # the saved record during client creation and discard the edited connection settings. + preview_server_id: Final = ( + "" + if request.auth_type in _STAGED_AUTH_VALUE_AUTH_TYPES or request.auth_type in (None, MCPAuth.none) + else request.server_id or "" + ) server_model: Final = MCPServer( - server_id=request.server_id or "", + server_id=preview_server_id, name=request.alias or request.server_name or "", url=request.url, transport=request.transport, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 3487f634251..771a73c8ad0 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -87,6 +87,72 @@ def _route_has_dependency(route, dependency) -> bool: class TestExecuteWithMcpClient: + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("auth_type", "auth_value", "expected_auth"), + ( + (MCPAuth.none, None, {}), + (MCPAuth.basic, "preview:correct", {"Authorization": "Basic cHJldmlldzpjb3JyZWN0"}), + (MCPAuth.basic, None, {"Authorization": "Basic cHJldmlldzpzdG9yZWQ="}), + (MCPAuth.bearer_token, "edited", {"Authorization": "Bearer edited"}), + (MCPAuth.api_key, "edited", {"X-API-Key": "edited"}), + (MCPAuth.token, "edited", {"Authorization": "token edited"}), + (MCPAuth.authorization, "Custom edited", {"Authorization": "Custom edited"}), + ), + ) + async def test_static_preview_uses_edited_connection_instead_of_registered_server( + self, + monkeypatch: pytest.MonkeyPatch, + auth_type: MCPAuth, + auth_value: str | None, + expected_auth: dict[str, str], + ) -> None: + from starlette.datastructures import Headers + + from litellm.experimental_mcp_client.client import MCPClient + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.management_endpoints import mcp_management_endpoints + + saved: Final = MCPServer( + server_id="saved-preview-server", + name="saved", + url="https://stored.example/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.basic, + authentication_token="preview:stored", + ) + manager: Final = MCPServerManager() + manager.registry = {saved.server_id: saved} + monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", manager) + monkeypatch.setattr(mcp_management_endpoints, "global_mcp_server_manager", manager) + payload: Final = NewMCPServerRequest( + server_id=saved.server_id, + server_name="edited", + url="https://edited.example/mcp", + transport=MCPTransport.sse, + auth_type=auth_type, + credentials={"auth_value": auth_value} if auth_value is not None else None, + static_headers={"X-Preview": "edited"}, + ) + staged: Final = rest_endpoints._stage_server_test(payload, Headers()) + + async def inspect_connection(client: MCPClient) -> dict[str, object]: + return {"url": client.server_url, "transport": client.transport_type, "headers": client._get_auth_headers()} + + result: Final = await rest_endpoints._execute_with_mcp_client( + staged.request, + inspect_connection, + mcp_auth_header=staged.mcp_auth_header, + oauth2_headers=staged.oauth2_headers, + ) + assert result == { + "url": "https://edited.example/mcp", + "transport": MCPTransport.sse, + "headers": {"X-Preview": "edited", **expected_auth}, + } + assert manager.get_mcp_server_by_id(saved.server_id) is saved + assert saved.url == "https://stored.example/mcp" + @pytest.mark.asyncio async def test_redacts_stack_trace(self, monkeypatch): async def fake_create_client(*args, **kwargs): From b554d79914ea0b0a6f12ad17d944d064c3a8f689 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:01:11 -0700 Subject: [PATCH 264/411] fix(cli): resolve Claude Code's settings file through CLAUDE_CONFIG_DIR Claude Code reads settings.json from CLAUDE_CONFIG_DIR when it is set, while lite login --config-claude wrote to ~/.claude/settings.json and lite claude checked that same file for its apiKeyHelper. With the override set, lite could drop ANTHROPIC_AUTH_TOKEN because the helper lives in a file Claude Code never reads, leaving it with no key at all. claude_settings_path(environ) now picks the file the way Claude Code does, and both commands go through it. The CLI test directory gets an autouse conftest that isolates HOME, USERPROFILE, and CLAUDE_CONFIG_DIR per test and fails any test that writes the developer's real settings.json, restoring it first. --- litellm/proxy/client/cli/commands/agents.py | 14 ++++--- litellm/proxy/client/cli/commands/auth.py | 10 +++-- .../client/cli/commands/claude_settings.py | 11 +++++ .../test_litellm/proxy/client/cli/conftest.py | 32 ++++++++++++++ .../proxy/client/cli/test_agents.py | 42 +++++++++++++++---- .../proxy/client/cli/test_auth_commands.py | 7 ++-- .../proxy/client/cli/test_claude_settings.py | 17 ++++++++ .../proxy/client/cli/test_up_commands.py | 7 +--- 8 files changed, 115 insertions(+), 25 deletions(-) create mode 100644 tests/test_litellm/proxy/client/cli/conftest.py diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index 8903be8ca5c..ab99754fdc7 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -4,6 +4,7 @@ import subprocess import sys from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from pathlib import Path from types import MappingProxyType from typing import Final, TypeAlias @@ -12,7 +13,7 @@ import requests from pydantic import BaseModel, TypeAdapter, ValidationError from .auth import CliContextObj, context_secret_vault, get_stored_api_key, login -from .claude_settings import CLAUDE_SETTINGS_PATH, lite_api_key_helper_configured +from .claude_settings import claude_settings_path, lite_api_key_helper_configured from .cmd_quoting import quote_for_cmd from .pi import ( LITELLM_PROXY_API_KEY_ENV, @@ -542,10 +543,12 @@ def resolve_api_key(ctx: click.Context) -> str: _SKIP_VERIFY_HELP: Final = "Skip the pre-launch key check against the proxy." -def _helper_supplies_token(ctx_obj: CliContextObj, base_url: str, profiles: frozenset[str]) -> bool: +def _helper_supplies_token( + ctx_obj: CliContextObj, base_url: str, profiles: frozenset[str], settings_path: Path +) -> bool: if PROFILE_ANTHROPIC not in profiles or not ctx_obj.get("api_key_from_token_file"): return False - return lite_api_key_helper_configured(base_url, CLAUDE_SETTINGS_PATH) + return lite_api_key_helper_configured(base_url, settings_path) def _launch(ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify: bool) -> None: @@ -555,10 +558,11 @@ def _launch(ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify api_key: Final = resolve_api_key(ctx) display_name, profiles = agent_profile(binary) - helper_supplies_token: Final = _helper_supplies_token(ctx_obj, base_url, profiles) + settings_path: Final = claude_settings_path(os.environ) + helper_supplies_token: Final = _helper_supplies_token(ctx_obj, base_url, profiles, settings_path) click.echo(f"litellm: routing {display_name} through proxy at {base_url.rstrip('/')}") if helper_supplies_token: - click.echo(f"litellm: {display_name} reads its key from the apiKeyHelper in {CLAUDE_SETTINGS_PATH}") + click.echo(f"litellm: {display_name} reads its key from the apiKeyHelper in {settings_path}") try: run_agent( diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 12a288202b6..8e766d94732 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -1,3 +1,4 @@ +import os import sys import time import webbrowser @@ -40,9 +41,9 @@ from litellm.litellm_core_utils.cli_token_utils import ( ) from .claude_settings import ( - CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS, ClaudeSettingsError, + claude_settings_path, write_claude_settings, ) from .pkce_login import ( @@ -778,12 +779,13 @@ def _render_and_prompt_for_team_selection(teams: list[CliTeam]) -> str | None: def _configure_claude_code(base_url: str) -> None: - """Point Claude Code at base_url by patching ~/.claude/settings.json.""" + """Point Claude Code at base_url by patching the settings.json it reads.""" + settings_path: Final = claude_settings_path(os.environ) try: - write_claude_settings(base_url, CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS) + write_claude_settings(base_url, settings_path, SETTINGS_FILE_OWNERS) except ClaudeSettingsError as e: raise click.ClickException(f"Logged in, but could not configure Claude Code: {e}") - click.echo(f"\nConfigured Claude Code: {CLAUDE_SETTINGS_PATH} now routes through {base_url.rstrip('/')}.") + click.echo(f"\nConfigured Claude Code: {settings_path} now routes through {base_url.rstrip('/')}.") click.echo("Your other Claude Code settings were left untouched. Restart Claude Code to pick this up.") diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index bc58c2ed815..524edcb899a 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -30,6 +30,7 @@ ENABLE_GATEWAY_MODEL_DISCOVERY_KEY: Final = "CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DI ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE: Final = "1" CLAUDE_SETTINGS_PATH: Final = Path.home() / ".claude" / "settings.json" +CLAUDE_CONFIG_DIR_ENV: Final = "CLAUDE_CONFIG_DIR" BACKUP_PATH: Final = Path.home() / ".litellm" / "claude_settings_backup.json" AUTOROUTE_BACKUP_PATH: Final = Path.home() / ".litellm" / "autorouter" / "claude_settings_backup.json" @@ -55,6 +56,14 @@ class ClaudeSettingsError(Exception): """Raised for any user-actionable failure while reading or writing Claude Code settings.""" +def claude_settings_path(environ: Mapping[str, str]) -> Path: + """The settings.json Claude Code reads: under CLAUDE_CONFIG_DIR when set, else ~/.claude/settings.json.""" + config_dir: Final = environ.get(CLAUDE_CONFIG_DIR_ENV, "") + if not config_dir: + return CLAUDE_SETTINGS_PATH + return Path(config_dir).expanduser() / "settings.json" + + def load_json_or_empty(path: Path) -> dict[str, JsonValue]: try: content: Final = path.read_bytes() if path.exists() else b"" @@ -173,6 +182,7 @@ __all__ = ( "API_KEY_HELPER_KEY", "AUTOROUTE_BACKUP_PATH", "BACKUP_PATH", + "CLAUDE_CONFIG_DIR_ENV", "CLAUDE_SETTINGS_PATH", "ENABLE_GATEWAY_MODEL_DISCOVERY_KEY", "ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE", @@ -182,6 +192,7 @@ __all__ = ( "SETTINGS_FILE_OWNERS", "ClaudeSettingsError", "SettingsFileOwner", + "claude_settings_path", "lite_api_key_helper_configured", "load_json_or_empty", "merge_claude_settings", diff --git a/tests/test_litellm/proxy/client/cli/conftest.py b/tests/test_litellm/proxy/client/cli/conftest.py new file mode 100644 index 00000000000..50c76d3f125 --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/conftest.py @@ -0,0 +1,32 @@ +import os +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import pytest + +REAL_CLAUDE_SETTINGS: Final = Path(os.path.expanduser("~")) / ".claude" / "settings.json" + + +def _current_bytes() -> bytes | None: + return REAL_CLAUDE_SETTINGS.read_bytes() if REAL_CLAUDE_SETTINGS.exists() else None + + +@pytest.fixture(autouse=True) +def isolated_claude_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: + before: Final = _current_bytes() + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / ".claude")) + yield tmp_path + after: Final = _current_bytes() + if after == before: + return + if before is None: + REAL_CLAUDE_SETTINGS.unlink() + else: + REAL_CLAUDE_SETTINGS.write_bytes(before) + pytest.fail( + f"this test wrote the developer's real {REAL_CLAUDE_SETTINGS}; the original bytes were restored. " + "Resolve the Claude settings path at call time (never Path.home() at import) and point the test at tmp_path" + ) diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py index 894c2406622..bebea285edc 100644 --- a/tests/test_litellm/proxy/client/cli/test_agents.py +++ b/tests/test_litellm/proxy/client/cli/test_agents.py @@ -1093,20 +1093,48 @@ class TestAgentCommands: in result.output ) - def _invoke_claude_with_settings(self, tmp_path, settings, obj): - settings_path = tmp_path / "settings.json" + def _invoke_claude_with_settings(self, tmp_path, settings, obj, *, default_settings=None): + config_dir = tmp_path / "claude-config" + config_dir.mkdir() if settings is not None: - settings_path.write_text(json.dumps(settings)) + (config_dir / "settings.json").write_text(json.dumps(settings)) + default_path = tmp_path / "home-claude" / "settings.json" + default_path.parent.mkdir() + if default_settings is not None: + default_path.write_text(json.dumps(default_settings)) captured = {} with ( - patch(f"{AGENTS_MODULE}.CLAUDE_SETTINGS_PATH", settings_path), + patch(f"{CLAUDE_SETTINGS_MODULE}.CLAUDE_SETTINGS_PATH", default_path), patch(f"{CLAUDE_SETTINGS_MODULE}.shutil.which", return_value="/usr/local/bin/lite"), patch(f"{AGENTS_MODULE}.run_agent", side_effect=lambda b, k, c, **kw: captured.update(kw)), ): - result = self.runner.invoke(_agent_command("claude"), [], obj=obj) + result = self.runner.invoke( + _agent_command("claude"), [], obj=obj, env={"CLAUDE_CONFIG_DIR": str(config_dir)} + ) assert result.exit_code == 0, result.output return captured, result.output + def test_helper_is_read_from_the_config_dir_claude_code_uses(self, tmp_path): + captured, output = self._invoke_claude_with_settings( + tmp_path, + {"apiKeyHelper": "/usr/local/bin/lite --base-url http://localhost:4000 auth print-token"}, + {"base_url": "http://localhost:4000", "api_key": "sk-key", "api_key_from_token_file": True}, + ) + + assert captured["export_anthropic_token"] is False + assert str(tmp_path / "claude-config" / "settings.json") in output + + def test_helper_only_in_the_default_file_keeps_the_env_token_when_config_dir_points_elsewhere(self, tmp_path): + captured, output = self._invoke_claude_with_settings( + tmp_path, + None, + {"base_url": "http://localhost:4000", "api_key": "sk-key", "api_key_from_token_file": True}, + default_settings={"apiKeyHelper": "/usr/local/bin/lite --base-url http://localhost:4000 auth print-token"}, + ) + + assert captured["export_anthropic_token"] is True + assert "apiKeyHelper" not in output + def test_stored_login_with_a_matching_helper_leaves_the_token_to_the_helper(self, tmp_path): captured, output = self._invoke_claude_with_settings( tmp_path, @@ -1145,11 +1173,9 @@ class TestAgentCommands: assert captured["export_anthropic_token"] is True - def test_codex_never_consults_claude_settings(self, tmp_path): - settings_path = tmp_path / "settings.json" + def test_codex_never_consults_claude_settings(self): captured = {} with ( - patch(f"{AGENTS_MODULE}.CLAUDE_SETTINGS_PATH", settings_path), patch(f"{AGENTS_MODULE}.lite_api_key_helper_configured", side_effect=AssertionError("consulted")), patch(f"{AGENTS_MODULE}.run_agent", side_effect=lambda b, k, c, **kw: captured.update(kw)), ): diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 821323e722c..225396d78f6 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -1416,7 +1416,6 @@ class TestLoginConfigClaude: patch("requests.get", return_value=poll_response), patch("litellm.proxy.client.cli.commands.auth.save_cli_token"), patch("litellm.proxy.client.cli.interface.show_commands"), - patch("litellm.proxy.client.cli.commands.auth.CLAUDE_SETTINGS_PATH", settings_path), patch( "litellm.proxy.client.cli.commands.auth.SETTINGS_FILE_OWNERS", (SettingsFileOwner(backup_path, "lite up", "lite down"),), @@ -1426,7 +1425,9 @@ class TestLoginConfigClaude: return_value="/usr/local/bin/lite", ), ): - result = self.runner.invoke(login, args, obj={"base_url": base_url}) + result = self.runner.invoke( + login, args, obj={"base_url": base_url}, env={"CLAUDE_CONFIG_DIR": str(settings_path.parent)} + ) return result, settings_path, backup_path def test_default_login_does_not_touch_claude_settings(self, tmp_path): @@ -1445,7 +1446,7 @@ class TestLoginConfigClaude: assert written["env"]["ANTHROPIC_BASE_URL"] == "https://test.example.com" assert written["env"]["ENABLE_TOOL_SEARCH"] == "true" assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://test.example.com auth print-token" - assert "Configured Claude Code" in result.output + assert f"Configured Claude Code: {settings_path} now routes through https://test.example.com." in result.output def test_flag_preserves_unrelated_settings_on_an_existing_file(self, tmp_path): settings_path = tmp_path / "claude" / "settings.json" diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py index 8a2bfea3bd7..0ff92567abc 100644 --- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py +++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py @@ -2,6 +2,7 @@ import json import shlex import stat import time +from pathlib import Path from unittest.mock import patch import pytest @@ -12,9 +13,11 @@ from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.claude_settings import ( AUTOROUTE_BACKUP_PATH, BACKUP_PATH, + CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS, ClaudeSettingsError, SettingsFileOwner, + claude_settings_path, lite_api_key_helper_configured, resolve_api_key_helper, write_claude_settings, @@ -360,6 +363,20 @@ class TestDoesNotDestroyUserOwnedStructure: assert json.loads(settings_path.read_text())["env"] == "not-an-object" +class TestClaudeSettingsPath: + def test_defaults_to_the_home_settings_file(self): + assert claude_settings_path({}) == CLAUDE_SETTINGS_PATH + assert claude_settings_path({"CLAUDE_CONFIG_DIR": ""}) == CLAUDE_SETTINGS_PATH + + def test_follows_claude_config_dir_like_claude_code_does(self, tmp_path): + assert claude_settings_path({"CLAUDE_CONFIG_DIR": str(tmp_path)}) == tmp_path / "settings.json" + + def test_expands_a_tilde_in_claude_config_dir(self): + assert claude_settings_path({"CLAUDE_CONFIG_DIR": "~/.claude-work"}) == ( + Path.home() / ".claude-work" / "settings.json" + ) + + class TestLiteApiKeyHelperConfigured: def _settings(self, tmp_path, payload): settings_path = tmp_path / "settings.json" diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index aead1764b0e..8e33e1ed350 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -620,10 +620,7 @@ class TestUpCanInvokeTheRealLoginCommand: ctx.obj = {"base_url": "http://127.0.0.1:9"} ctx.invoke(real_login, pkce=False) - with ( - patch(f"{AUTH_MODULE}.CLAUDE_SETTINGS_PATH", settings_path), - patch(f"{AUTH_MODULE}._start_cli_sso_flow", side_effect=RuntimeError("stop")), - ): - CliRunner().invoke(driver, [], standalone_mode=False) + with patch(f"{AUTH_MODULE}._start_cli_sso_flow", side_effect=RuntimeError("stop")): + CliRunner().invoke(driver, [], standalone_mode=False, env={"CLAUDE_CONFIG_DIR": str(tmp_path)}) assert not settings_path.exists() From 317b29e69d658ca2502e656d4a30c4db5ced0198 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 18:02:23 -0700 Subject: [PATCH 265/411] feat(cli): add lite configure claude and lite unconfigure claude (#40319) Persistently route Claude Code through a LiteLLM proxy with a long-lived virtual key or the stored lite login, turn on gateway model discovery so /model lists the proxy's models, optionally pick the model Claude Code starts on, and record what changed so unconfigure restores only the keys the user has not touched since. lite login --config-claude writes through the same receipt and is undoable too. The two settings merges (lite up / --config-claude and lite autoroute) collapse into one credential-aware merge --- litellm/proxy/client/cli/README.md | 22 +- litellm/proxy/client/cli/commands/agents.py | 5 +- litellm/proxy/client/cli/commands/auth.py | 29 +- .../client/cli/commands/autoroute/commands.py | 9 +- .../client/cli/commands/autoroute/settings.py | 51 -- .../client/cli/commands/claude_settings.py | 517 +++++++++++++++-- .../proxy/client/cli/commands/configure.py | 252 +++++++++ litellm/proxy/client/cli/commands/pi.py | 31 +- litellm/proxy/client/cli/commands/up.py | 9 +- litellm/proxy/client/cli/main.py | 4 + .../client/cli/autoroute/test_commands.py | 4 + .../client/cli/autoroute/test_settings.py | 63 --- .../proxy/client/cli/test_auth_commands.py | 40 +- .../proxy/client/cli/test_claude_settings.py | 535 +++++++++++++++++- .../client/cli/test_configure_commands.py | 331 +++++++++++ .../test_litellm/proxy/client/cli/test_pi.py | 26 +- .../proxy/client/cli/test_up_commands.py | 40 +- 17 files changed, 1718 insertions(+), 250 deletions(-) delete mode 100644 litellm/proxy/client/cli/commands/autoroute/settings.py create mode 100644 litellm/proxy/client/cli/commands/configure.py delete mode 100644 tests/test_litellm/proxy/client/cli/autoroute/test_settings.py create mode 100644 tests/test_litellm/proxy/client/cli/test_configure_commands.py diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index f7d9eb7da9a..03d01ff7f66 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -508,7 +508,7 @@ The credential is short-lived by design (default 24h, configurable via `LITELLM_ ### Route Every Claude Code Session Through the Proxy -`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL, `env.ENABLE_TOOL_SEARCH` to `true` and `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` to `1` when those keys are missing, and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it. +`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL, `env.ENABLE_TOOL_SEARCH` to `true` and `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` to `1` when those keys are missing, and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it. Two things need to already be true: you've run `lite login` (or `lite login --pkce`, whose key the helper renews on its own), since the apiKeyHelper depends on that stored token, and the proxy is already reachable, since `lite up` does not start one for you. @@ -532,12 +532,28 @@ Cursor is not supported: it has no equivalent file-based config to hot-patch thi lite --base-url https://your-proxy.example.com login --config-claude ``` -It writes the same settings `lite up` does, `env.ANTHROPIC_BASE_URL`, `env.ENABLE_TOOL_SEARCH`, `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY`, and `apiKeyHelper`, but persistently: there is no backup, nothing to restore, and no foreground process to keep alive. Every other key in `~/.claude/settings.json` is preserved, the file is created if it does not exist, and it is written atomically with owner-only permissions. Plain `lite login` is unchanged; nothing happens to your Claude Code config unless you pass the flag. +It writes the same settings `lite up` does, `env.ANTHROPIC_BASE_URL`, `env.ENABLE_TOOL_SEARCH`, `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY`, and `apiKeyHelper`, but persistently: no foreground process to keep alive, and `lite unconfigure claude` restores what it changed (see below). Every other key in `~/.claude/settings.json` is preserved, the file is created if it does not exist, and it is written atomically with owner-only permissions. Plain `lite login` is unchanged; nothing happens to your Claude Code config unless you pass the flag. Because the credential is reached through `apiKeyHelper` rather than copied into the file, a later `lite login` refreshes it with no further action: Claude Code re-runs the helper on every request and picks up whatever token the most recent login stored. Nothing secret is written to `settings.json`. Run it again to point Claude Code at a different proxy; the base URL and the helper are both rewritten. `lite up` and `--config-claude` manage the same file, so the flag refuses to run while a `lite up` session holds a backup, and tells you to run `lite down` first, rather than writing settings that `lite up` would silently revert when it stops. +#### Configuring Claude Code Once, With a Virtual Key or Your Login + +`lite configure claude` wires Claude Code up persistently and `lite unconfigure claude` puts things back. It is what `lite login --config-claude` does, plus a pinned model and an undo, and it also takes a long-lived virtual key when that is what you have: + +```bash +curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh +lite --base-url https://your-proxy.example.com configure claude --api-key sk-... --model claude-auto +claude +``` + +With `--api-key` (or `lite --api-key` / `LITELLM_PROXY_API_KEY`) the key is written into `env.ANTHROPIC_AUTH_TOKEN`. Without one, your `lite login` credential is used the way `--config-claude` uses it, through `apiKeyHelper`, so a later `lite login` (or a `--pkce` renewal) picks up on its own and nothing secret lands in the file; a missing or stale login is refreshed first. Either way the command checks the key against `GET /v1/models`, then patches `~/.claude/settings.json`: `env.ANTHROPIC_BASE_URL`, the credential, and `env.ENABLE_TOOL_SEARCH` and `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` when those are missing, so Claude Code's `/model` picker lists the proxy's models (the ones whose id contains `claude` or `anthropic`) and you pick between them as usual. Claude Code keeps its own default model until you switch, so that id has to exist on the proxy for the first message to go through; `--model` (or the interactive prompt below) sets the model Claude Code starts on instead, as the top-level `model` key, which has to be on `/v1/models` for the key. Nothing forces Claude Code's sub-agent or background tiers onto a proxy model, so those built-in ids need to exist on the proxy too; `lite autoroute up` is the mode that pins every tier to one group. Claude Code treats a name it does not know as an unknown model: it prints a one-line `unrecognized_model` note, assumes a 200k context window and sends no thinking parameters for it, so either name the group like a Claude model id or append `[1m]` to opt into the 1M window. The other credential slots (`env.ANTHROPIC_API_KEY`, a stale `env.ANTHROPIC_AUTH_TOKEN` or `apiKeyHelper`) are removed so they cannot fight the one written. Every other setting is preserved and the file is written atomically with owner-only permissions; if `settings.json` is a symlink into a dotfiles repository, the key is written through to that target and the command says so, so keep it out of version control + +Plain `lite configure`, with no agent named, asks the same things interactively: which agents to wire (Claude Code today) and which of the proxy's models to start on, picked from `/v1/models` with a type-to-filter prompt + +What the command changed is recorded in `~/.litellm/claude_configure_state.json` (previous values plus fingerprints of what was written, never a second copy of the key). `lite unconfigure claude` restores each of those keys only if it still holds what `configure` wrote, so anything you changed since is left alone and named in the output; a `settings.json` or `env` object that only existed because of `configure` is removed again. Ownership moves only by a write: running `configure` again (a re-login is one) refreshes the record only for the keys its merge changed, keeps the original snapshot of a key that still holds what it wrote, and snapshots afresh a key you changed in between, so `unconfigure` brings back whatever the repeat displaced and never adopts your edit as its own. A credential (`env.ANTHROPIC_API_KEY`, `env.ANTHROPIC_AUTH_TOKEN`, `apiKeyHelper`) is put back only when the restored file points at the `ANTHROPIC_BASE_URL` it was captured next to; otherwise it stays removed, the output says which server it belonged to, and the receipt is kept so pointing the URL back and running `unconfigure` again finishes the job. It also undoes `lite login --config-claude`, which writes through the same path. Like `--config-claude`, both refuse to run while a `lite up` or `lite autoroute up` session holds a backup, and that check comes before any login prompt or request + ### QA Complexity-Based Auto-Routing Against Your Real Proxy `lite autoroute` lets you try LiteLLM's complexity-based auto-routing -- picking a cheaper or more expensive model depending on how complex a prompt looks -- against models your key already has access to on your real, running proxy, without editing that proxy's `config.yaml` and without any real request ever bypassing it. It builds a second, throwaway proxy locally that forwards every request back to your real proxy, and points Claude Code at that local proxy for the duration of the session. @@ -584,7 +600,7 @@ An interactive wizard. It runs the same model-group discovery as above, splits t The wizard writes the result to `~/.litellm/autorouter/config.yaml` with `0600` permissions, since the file embeds your real proxy API key. Every model referenced anywhere in that config -- tier targets, the classifier model, the embedding model -- becomes its own `litellm_proxy/` deployment whose `api_base` and `api_key` point back at your real proxy. That is the trick that keeps your real proxy's config untouched: every actual network call this generates, whether it is the routed completion, an LLM-classifier call, or an embedding call, forwards transparently through your real, already-running proxy with your real key. -You do not need to tell Claude Code to request `autorouter` by name yourself: `lite autoroute up` also sets `ANTHROPIC_DEFAULT_SONNET_MODEL`, `ANTHROPIC_DEFAULT_HAIKU_MODEL`, and `ANTHROPIC_DEFAULT_OPUS_MODEL` to `autorouter` in `~/.claude/settings.json`, so every one of Claude Code's own model tiers requests it directly regardless of `/model` or whatever it defaults to otherwise. (A bare `model_name: "*"` deployment looks like the obvious way to catch any request instead, but litellm's Router looks up auto-router deployments by the literal requested model string with no wildcard resolution, so a `"*"` entry would never actually match real traffic -- these env var overrides are what makes it work.) +You do not need to tell Claude Code to request `autorouter` by name yourself: `lite autoroute up` also sets the top-level `model` and `ANTHROPIC_DEFAULT_SONNET_MODEL`, `ANTHROPIC_DEFAULT_HAIKU_MODEL`, `ANTHROPIC_DEFAULT_OPUS_MODEL` and `ANTHROPIC_DEFAULT_FABLE_MODEL` to `autorouter` in `~/.claude/settings.json` (and `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` to `1` when missing, like every other wiring), so every one of Claude Code's own model tiers requests it directly regardless of `/model` or whatever it defaults to otherwise. (A bare `model_name: "*"` deployment looks like the obvious way to catch any request instead, but litellm's Router looks up auto-router deployments by the literal requested model string with no wildcard resolution, so a `"*"` entry would never actually match real traffic -- these env var overrides are what makes it work.) You must run `configure` at least once before `up`; running `up` first fails with a clear error telling you to configure first. diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index 7a0ae9dc955..b287293eb7a 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -16,6 +16,7 @@ from .cmd_quoting import quote_for_cmd from .pi import ( LITELLM_PROXY_API_KEY_ENV, PI_PROVIDER_NAME, + ListingFailure, PiSyncError, fetch_model_ids, fetch_model_limits, @@ -165,7 +166,9 @@ def prepare_pi( """ ids: Final = fetch_model_ids(base_url, api_key, get=get) if isinstance(ids, PiSyncError): - raise AgentRunError(ids.message) + raise AgentRunError( + f"{ids.message} pi would have nothing to run." if ids.kind is ListingFailure.EMPTY else ids.message + ) limits: Final = fetch_model_limits(base_url, api_key, get=get) path: Final = models_json_path(base_env) error: Final = sync_models_json(path, base_url, ids, limits) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 12a288202b6..24b1c766664 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -41,9 +41,15 @@ from litellm.litellm_core_utils.cli_token_utils import ( from .claude_settings import ( CLAUDE_SETTINGS_PATH, + CONFIGURE_STATE_PATH, SETTINGS_FILE_OWNERS, + STARTING_MODEL_ROLE, + ApiKeyHelper, ClaudeSettingsError, - write_claude_settings, + KeepModel, + configure_claude_settings, + refuse_while_owned, + resolve_api_key_helper, ) from .pkce_login import ( Http, @@ -778,13 +784,23 @@ def _render_and_prompt_for_team_selection(teams: list[CliTeam]) -> str | None: def _configure_claude_code(base_url: str) -> None: - """Point Claude Code at base_url by patching ~/.claude/settings.json.""" + """Point Claude Code at base_url by patching ~/.claude/settings.json, undoable with `lite unconfigure claude`.""" try: - write_claude_settings(base_url, CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS) + configure_claude_settings( + base_url, + ApiKeyHelper(resolve_api_key_helper(base_url)), + KeepModel(), + CLAUDE_SETTINGS_PATH, + CONFIGURE_STATE_PATH, + SETTINGS_FILE_OWNERS, + ) except ClaudeSettingsError as e: raise click.ClickException(f"Logged in, but could not configure Claude Code: {e}") click.echo(f"\nConfigured Claude Code: {CLAUDE_SETTINGS_PATH} now routes through {base_url.rstrip('/')}.") - click.echo("Your other Claude Code settings were left untouched. Restart Claude Code to pick this up.") + click.echo( + "Your other Claude Code settings were left untouched. Restart Claude Code to pick this up. " + f"Undo with `lite unconfigure claude`; `lite configure claude --model` sets {STARTING_MODEL_ROLE}." + ) def _finish_login(base_url: str, api_key: str, config_claude: bool, stored: SecretSave) -> None: @@ -853,6 +869,11 @@ def login(ctx: click.Context, config_claude: bool, pkce: bool) -> None: ctx_obj: Final[CliContextObj] = ctx.obj base_url: Final = ctx_obj["base_url"] + if config_claude: + try: + refuse_while_owned(CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS) + except ClaudeSettingsError as e: + raise click.ClickException(f"Cannot configure Claude Code, so not logging in: {e}") try: if pkce: diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index 26d45138a27..86701f186fd 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -14,11 +14,13 @@ from ..claude_settings import ( AUTOROUTE_BACKUP_PATH, CLAUDE_SETTINGS_PATH, ClaudeSettingsError, + StaticToken, load_json_or_empty, + merge_claude_settings, ) from ..up import BackupRecord as ClaudeBackupRecord from ..up import restore_claude_settings, write_backup -from .config import master_key_from_config +from .config import AUTOROUTER_MODEL_NAME, master_key_from_config from .process import ( CONFIG_PATH, DEFAULT_AUTOROUTE_PORT, @@ -37,7 +39,6 @@ from .process import ( terminate, write_pid_record, ) -from .settings import merge_claude_settings_static_token from .wizard import run_configure_wizard _GENERATED_CONFIG_ADAPTER: Final = TypeAdapter(dict[str, JsonValue]) @@ -156,7 +157,9 @@ def up(port: int) -> None: ClaudeBackupRecord(existed=original_existed, content=original_settings if original_existed else None), AUTOROUTE_BACKUP_PATH, ) - merged: Final = merge_claude_settings_static_token(original_settings, base_url, master_key) + merged: Final = merge_claude_settings( + original_settings, base_url, StaticToken(master_key), AUTOROUTER_MODEL_NAME, AUTOROUTER_MODEL_NAME + ) CLAUDE_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) with secure_create(CLAUDE_SETTINGS_PATH) as f: json.dump(merged, f, indent=2) diff --git a/litellm/proxy/client/cli/commands/autoroute/settings.py b/litellm/proxy/client/cli/commands/autoroute/settings.py deleted file mode 100644 index 60729b5410d..00000000000 --- a/litellm/proxy/client/cli/commands/autoroute/settings.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import Final - -from pydantic import JsonValue - -from .config import AUTOROUTER_MODEL_NAME - -ENV_KEY: Final = "env" -API_KEY_HELPER_KEY: Final = "apiKeyHelper" -ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY" -ANTHROPIC_AUTH_TOKEN_KEY: Final = "ANTHROPIC_AUTH_TOKEN" -ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL" -ENABLE_TOOL_SEARCH_KEY: Final = "ENABLE_TOOL_SEARCH" -ENABLE_TOOL_SEARCH_VALUE: Final = "true" -# Force every one of Claude Code's own model tiers to request the auto-router by name. -# Router's auto-router registry is keyed by the literal requested model string -# (litellm/router.py:10711-10717) with no wildcard/pattern resolution, so a bare "*" -# model_name can never work as a catch-all -- these overrides are what actually makes -# Claude Code send "autorouter" regardless of /model or its own version-specific defaults. -ANTHROPIC_DEFAULT_MODEL_ENV_KEYS: Final = ( - "ANTHROPIC_DEFAULT_SONNET_MODEL", - "ANTHROPIC_DEFAULT_HAIKU_MODEL", - "ANTHROPIC_DEFAULT_OPUS_MODEL", -) - - -def merge_claude_settings_static_token( - settings: dict[str, JsonValue], base_url: str, auth_token: str -) -> dict[str, JsonValue]: - """Return a new settings dict wired to a local ephemeral proxy with a static token. - - Unlike up.py's merge_claude_settings (which sets apiKeyHelper for a long-lived, real - remote proxy needing refreshable SSO tokens), this proxy is ephemeral and its key is the - locally persisted autoroute master key, so a plain env var is simpler and correct. Any - existing apiKeyHelper is cleared so it can't fight with the static token. - """ - raw_env: Final = settings.get(ENV_KEY, {}) - base_env: Final = raw_env if isinstance(raw_env, dict) else {} - env: Final[dict[str, JsonValue]] = { - ENABLE_TOOL_SEARCH_KEY: ENABLE_TOOL_SEARCH_VALUE, - **base_env, - ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), - ANTHROPIC_AUTH_TOKEN_KEY: auth_token, - **{key: AUTOROUTER_MODEL_NAME for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS}, - } - env.pop(ANTHROPIC_API_KEY_KEY, None) - merged: Final[dict[str, JsonValue]] = {**settings, ENV_KEY: env} - merged.pop(API_KEY_HELPER_KEY, None) - return merged - - -__all__ = ["merge_claude_settings_static_token"] diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index 5e3ce95f088..bd9cf85dc59 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -1,37 +1,70 @@ """Shared handling of Claude Code's ~/.claude/settings.json. -`lite up` patches this file temporarily and restores it on exit; `lite login ---config-claude` patches it persistently. Both need the same merge and the same -apiKeyHelper command, and `up` already imports from `auth`, so the shared parts -live here rather than in either command module. +`lite up` and `lite autoroute up` patch this file temporarily and restore it on +exit; `lite login --config-claude` and `lite configure claude` patch it +persistently and record how to undo it. All of them need the same merge and the +same apiKeyHelper command, and `up` already imports from `auth`, so the shared +parts live here rather than in any one command module. """ +import hashlib +import json import shlex import shutil import sys -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from functools import reduce +from itertools import chain from pathlib import Path -from typing import Final +from types import MappingProxyType +from typing import Final, TypeAlias -from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError -from litellm.litellm_core_utils.private_json import write_private_json +from litellm.litellm_core_utils.private_json import ( + commit_staged_json, + discard_staged_json, + ensure_private_dir, + stage_private_json, +) from .cmd_quoting import quote_for_cmd ENV_KEY: Final = "env" API_KEY_HELPER_KEY: Final = "apiKeyHelper" +MODEL_KEY: Final = "model" ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL" +ANTHROPIC_AUTH_TOKEN_KEY: Final = "ANTHROPIC_AUTH_TOKEN" ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY" ENABLE_TOOL_SEARCH_KEY: Final = "ENABLE_TOOL_SEARCH" ENABLE_TOOL_SEARCH_VALUE: Final = "true" ENABLE_GATEWAY_MODEL_DISCOVERY_KEY: Final = "CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY" ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE: Final = "1" +ANTHROPIC_DEFAULT_MODEL_ENV_KEYS: Final = ( + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + "ANTHROPIC_DEFAULT_FABLE_MODEL", +) +OWNED_ENV_KEYS: Final = ( + ENABLE_TOOL_SEARCH_KEY, + ENABLE_GATEWAY_MODEL_DISCOVERY_KEY, + ANTHROPIC_BASE_URL_KEY, + ANTHROPIC_AUTH_TOKEN_KEY, + ANTHROPIC_API_KEY_KEY, +) +OWNED_TOP_LEVEL_KEYS: Final = (API_KEY_HELPER_KEY, MODEL_KEY) +OWNED_PATHS: Final = (*(f"{ENV_KEY}.{key}" for key in OWNED_ENV_KEYS), *OWNED_TOP_LEVEL_KEYS) +_CREDENTIAL_ENV_KEYS: Final = frozenset((ANTHROPIC_API_KEY_KEY, ANTHROPIC_AUTH_TOKEN_KEY)) +_CREDENTIAL_PATHS: Final = (*(f"{ENV_KEY}.{key}" for key in sorted(_CREDENTIAL_ENV_KEYS)), API_KEY_HELPER_KEY) +_BASE_URL_PATH: Final = f"{ENV_KEY}.{ANTHROPIC_BASE_URL_KEY}" +STARTING_MODEL_ROLE: Final = "the /model picker's default row, the model Claude Code starts on" CLAUDE_SETTINGS_PATH: Final = Path.home() / ".claude" / "settings.json" BACKUP_PATH: Final = Path.home() / ".litellm" / "claude_settings_backup.json" AUTOROUTE_BACKUP_PATH: Final = Path.home() / ".litellm" / "autorouter" / "claude_settings_backup.json" +CONFIGURE_STATE_PATH: Final = Path.home() / ".litellm" / "claude_configure_state.json" @dataclass(frozen=True, slots=True) @@ -55,6 +88,102 @@ class ClaudeSettingsError(Exception): """Raised for any user-actionable failure while reading or writing Claude Code settings.""" +@dataclass(frozen=True, slots=True) +class StaticToken: + """A long-lived virtual key, written into env.ANTHROPIC_AUTH_TOKEN.""" + + token: str + + +@dataclass(frozen=True, slots=True) +class ApiKeyHelper: + """A `lite auth print-token` command Claude Code runs per request, so a login renews in place.""" + + command: str + + +ClaudeCredential: TypeAlias = StaticToken | ApiKeyHelper + + +@dataclass(frozen=True, slots=True) +class KeepModel: + """Leave the top-level `model` as it is, the user's or an earlier configure's (a re-login).""" + + +@dataclass(frozen=True, slots=True) +class UnpinModel: + """Let go of a `model` an earlier configure pinned; one the user set themselves stays.""" + + +@dataclass(frozen=True, slots=True) +class StartOn: + """Pin the top-level `model`, the row Claude Code starts on.""" + + model: str + + +ModelChoice: TypeAlias = KeepModel | UnpinModel | StartOn + + +class OwnedValue(BaseModel): + """What one key held at a moment in time; `present=False` is an absent key, not a null one.""" + + model_config = ConfigDict(frozen=True) + + present: bool + value: JsonValue = None + + +class ConfigureReceipt(BaseModel): + """What `lite configure claude` found and what it owns, keyed by dotted path (`env.X` or a top-level key). + + Ownership moves only by a write: `written` fingerprints the keys some configure changed, at the + value it wrote; a repeat configure refreshes a fingerprint only for a key its merge changed and + carries the earlier one otherwise, so a key the user edited in between stops matching and is left + alone. `previous` is what each key held before configure took it over; a repeat keeps the earlier + snapshot while the key still holds our value and snapshots afresh otherwise, so whatever the + repeat displaces is what comes back. `endpoints` is the ANTHROPIC_BASE_URL each credential slot + was captured beside, so a credential is only ever put back next to the server it was issued for. + No fingerprint is a second copy of a token. + """ + + model_config = ConfigDict(frozen=True) + + file_existed: bool + env_present: bool + env_was_object: bool + previous: Mapping[str, OwnedValue] + written: Mapping[str, str] + endpoints: Mapping[str, OwnedValue] + + +@dataclass(frozen=True, slots=True) +class WithheldCredential: + """A credential left removed: captured beside `endpoint`, while the restored file points elsewhere.""" + + key: str + endpoint: str + + +@dataclass(frozen=True, slots=True) +class UnconfigureOutcome: + """Keys whose value unconfigure changed back, keys the user changed since and so were left as they + are, credentials withheld (the receipt is kept for them, so a later unconfigure can finish once the + URL points back), and whether no settings file remains.""" + + restored: tuple[str, ...] + kept: tuple[str, ...] + withheld: tuple[WithheldCredential, ...] = () + file_removed: bool = False + + +@dataclass(frozen=True, slots=True) +class _Claim: + previous: OwnedValue + written: str | None + endpoint: OwnedValue | None + + def load_json_or_empty(path: Path) -> dict[str, JsonValue]: try: content: Final = path.read_bytes() if path.exists() else b"" @@ -70,29 +199,104 @@ def load_json_or_empty(path: Path) -> dict[str, JsonValue]: ) -def merge_claude_settings( - settings: Mapping[str, JsonValue], base_url: str, api_key_helper: str -) -> dict[str, JsonValue]: - """Return a new settings dict wired to route Claude Code through the proxy. +def _env_object(settings: Mapping[str, JsonValue], path: Path) -> Mapping[str, JsonValue]: + raw_env: Final = settings.get(ENV_KEY) + if raw_env is None: + return MappingProxyType({}) + if not isinstance(raw_env, dict): + raise ClaudeSettingsError( + f'{path} has a non-object "{ENV_KEY}" value, which this would discard. Fix or remove it, then retry.' + ) + return raw_env - Only env.ANTHROPIC_BASE_URL and the top-level apiKeyHelper are overridden; a - stray env.ANTHROPIC_API_KEY is dropped so it cannot outrank the helper-issued - token (same reasoning as build_agent_env in agents.py). ENABLE_TOOL_SEARCH - defaults to true because Claude Code turns tool search off when - ANTHROPIC_BASE_URL is not a first-party Anthropic host, and - CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY defaults to 1 so the /model picker - is filled from the proxy's /v1/models; existing values of both are left - alone. Every other key is preserved untouched. + +def refuse_while_owned(settings_path: Path, owners: Sequence[SettingsFileOwner]) -> None: + """Refuse while `lite up` or `lite autoroute up` holds a backup it will restore over any write; a + purely local check, so commands run it before any login prompt or request.""" + for owner in owners: + if owner.backup_path.exists(): + raise ClaudeSettingsError( + f"`{owner.start_command}` is currently managing {settings_path} (backup at " + f"{owner.backup_path}) and will restore it when it stops. " + f"Run `{owner.stop_command}` first, then retry." + ) + + +def _write_target(settings_path: Path) -> Path: + """Write through a symlinked settings.json rather than replacing the link, which would silently + detach a file symlinked into a dotfiles repo.""" + try: + return settings_path.resolve() if settings_path.is_symlink() else settings_path + except OSError as e: + raise ClaudeSettingsError(f"Could not resolve {settings_path}: {e}") from e + + +def _stage(path: Path, document: Mapping[str, object]) -> str: + try: + return stage_private_json(str(path), document) + except OSError as e: + raise ClaudeSettingsError(f"Could not write {path}: {e}") from e + + +def _land( + path: Path, + staged: str | None, + also_discard: Sequence[str | None] = (), + commit: Callable[[str, str], None] = commit_staged_json, +) -> None: + """Commit a staged file to `path`, or remove `path` when nothing is staged for it. The one place a + filesystem error becomes a ClaudeSettingsError; on failure the operation's other staged files are + discarded, so no temp file holding a token is left behind.""" + try: + if staged is None: + path.unlink(missing_ok=True) + else: + commit(staged, str(path)) + except OSError as e: + for other in also_discard: + if other is not None: + discard_staged_json(other) + raise ClaudeSettingsError(f"Could not {'remove' if staged is None else 'write'} {path}: {e}") from e + + +def merge_claude_settings( + settings: Mapping[str, JsonValue], + base_url: str, + credential: ClaudeCredential, + default_model: str | None = None, + tier_model: str | None = None, +) -> Mapping[str, JsonValue]: + """Return a new settings mapping wired to route Claude Code through the proxy. + + A StaticToken lands in env.ANTHROPIC_AUTH_TOKEN, an ApiKeyHelper in the top-level apiKeyHelper; + the other credential slots are removed either way, since Claude Code given two credentials may + send the wrong one. ENABLE_TOOL_SEARCH and CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY get their + defaults only when missing. `default_model` is the top-level `model`, the row Claude Code starts + on; `tier_model` is `lite autoroute up`'s knob that points every ANTHROPIC_DEFAULT_*_MODEL at one + group. Apart from those tier keys, exactly OWNED_PATHS are touched. """ raw_env: Final = settings.get(ENV_KEY, {}) - base_env: Final = raw_env if isinstance(raw_env, dict) else {} - env: Final = { - ENABLE_TOOL_SEARCH_KEY: ENABLE_TOOL_SEARCH_VALUE, - ENABLE_GATEWAY_MODEL_DISCOVERY_KEY: ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE, - **{key: value for key, value in base_env.items() if key != ANTHROPIC_API_KEY_KEY}, - ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), - } - return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper} + current_env: Final = raw_env if isinstance(raw_env, dict) else {} + env: Final = dict( # mutable-ok: JSON document handed to json.dump, which rejects a read-only mapping + chain( + ( + (ENABLE_TOOL_SEARCH_KEY, ENABLE_TOOL_SEARCH_VALUE), + (ENABLE_GATEWAY_MODEL_DISCOVERY_KEY, ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE), + ), + ((key, value) for key, value in current_env.items() if key not in _CREDENTIAL_ENV_KEYS), + ((ANTHROPIC_BASE_URL_KEY, base_url.rstrip("/")),), + ((ANTHROPIC_AUTH_TOKEN_KEY, credential.token),) if isinstance(credential, StaticToken) else (), + ((key, tier_model) for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS if tier_model is not None), + ) + ) + return dict( # mutable-ok: JSON document handed to json.dump, which rejects a read-only mapping + chain( + ((key, value) for key, value in settings.items() if key not in (API_KEY_HELPER_KEY, ENV_KEY)), + ((ENV_KEY, env),), + ((API_KEY_HELPER_KEY, credential.command),) if isinstance(credential, ApiKeyHelper) else (), + ((MODEL_KEY, default_model),) if default_model is not None else (), + ) + ) def resolve_api_key_helper(base_url: str, platform: str = sys.platform) -> str: @@ -121,56 +325,255 @@ def resolve_api_key_helper(base_url: str, platform: str = sys.platform) -> str: return " ".join(quote(token) for token in (lite_path, "--base-url", base_url, "auth", "print-token")) -def write_claude_settings(base_url: str, settings_path: Path, owners: Sequence[SettingsFileOwner]) -> None: - """Persistently point Claude Code at base_url, preserving every unrelated setting. +def _owned(container: Mapping[str, JsonValue], key: str) -> OwnedValue: + return OwnedValue(present=key in container, value=container.get(key)) - Refuses while any owner holds a backup: each restores its backup when it - stops, which would silently undo this write. - """ - for owner in owners: - if owner.backup_path.exists(): - raise ClaudeSettingsError( - f"`{owner.start_command}` is currently managing {settings_path} (backup at " - f"{owner.backup_path}) and will restore it when it stops. " - f"Run `{owner.stop_command}` first, then retry." - ) - normalized_base_url: Final = base_url.rstrip("/") - api_key_helper: Final = resolve_api_key_helper(normalized_base_url) - existing: Final = load_json_or_empty(settings_path) - raw_env: Final = existing.get(ENV_KEY) - if raw_env is not None and not isinstance(raw_env, dict): - raise ClaudeSettingsError( - f'{settings_path} has a non-object "{ENV_KEY}" value, which this would discard. ' - "Fix or remove it, then retry." - ) - merged: Final = merge_claude_settings(existing, normalized_base_url, api_key_helper) - # os.replace() swaps the symlink itself for a regular file, silently detaching a - # settings.json that is symlinked into a dotfiles repo. There is no backup to undo - # that here, unlike `lite up`, so write through to the link's target instead. - target: Final = settings_path.resolve() if settings_path.is_symlink() else settings_path + +def _fingerprint(owned: OwnedValue) -> str: + return hashlib.sha256(json.dumps(owned.model_dump(mode="json"), sort_keys=True).encode()).hexdigest() + + +def _env(settings: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + raw_env: Final = settings.get(ENV_KEY) + return raw_env if isinstance(raw_env, dict) else MappingProxyType({}) + + +def _lookup(settings: Mapping[str, JsonValue], path: str) -> OwnedValue: + section, _, key = path.rpartition(".") + return _owned(_env(settings) if section else settings, key) + + +def _with_key(container: Mapping[str, JsonValue], key: str, owned: OwnedValue) -> Mapping[str, JsonValue]: + return dict( # mutable-ok: JSON document handed to json.dump, which rejects a read-only mapping + chain(((k, v) for k, v in container.items() if k != key), ((key, owned.value),) if owned.present else ()) + ) + + +def _with(settings: Mapping[str, JsonValue], path: str, owned: OwnedValue) -> Mapping[str, JsonValue]: + """`settings` with the key at `path` set (or removed when `owned` is absent); nothing else changes.""" + section, _, key = path.rpartition(".") + if not section: + return _with_key(settings, key, owned) + return _with_key(settings, section, OwnedValue(present=True, value=_with_key(_env(settings), key, owned))) + + +def _with_all(settings: Mapping[str, JsonValue], updates: Mapping[str, OwnedValue]) -> Mapping[str, JsonValue]: + return reduce(lambda acc, item: _with(acc, *item), updates.items(), settings) + + +def _ours(settings: Mapping[str, JsonValue], path: str, receipt: ConfigureReceipt) -> bool: + """Whether the key still holds what a configure wrote (a key no configure ever changed is never ours).""" + return receipt.written.get(path) == _fingerprint(_lookup(settings, path)) + + +def _claim( + path: str, + current: Mapping[str, JsonValue], + merged: Mapping[str, JsonValue], + earlier: ConfigureReceipt | None, + url_now: OwnedValue, +) -> _Claim: + """What this configure records for one key; see ConfigureReceipt for the rules.""" + before, after = _lookup(current, path), _lookup(merged, path) + carried: Final = earlier if earlier is not None and _ours(current, path, earlier) else None + return _Claim( + previous=before if carried is None else carried.previous.get(path, before), + written=_fingerprint(after) if before != after else (None if earlier is None else earlier.written.get(path)), + endpoint=None + if path not in _CREDENTIAL_PATHS + else (url_now if carried is None else carried.endpoints.get(path, url_now)), + ) + + +def _receipt( + current: Mapping[str, JsonValue], + merged: Mapping[str, JsonValue], + earlier: ConfigureReceipt | None, + file_exists: bool, +) -> ConfigureReceipt: + url_now: Final = _lookup(current, _BASE_URL_PATH) + claims: Final = MappingProxyType({path: _claim(path, current, merged, earlier, url_now) for path in OWNED_PATHS}) + return ConfigureReceipt( + file_existed=file_exists if earlier is None else earlier.file_existed, + env_present=ENV_KEY in current if earlier is None else earlier.env_present, + env_was_object=isinstance(current.get(ENV_KEY), dict) if earlier is None else earlier.env_was_object, + previous=MappingProxyType({path: claim.previous for path, claim in claims.items()}), + written=MappingProxyType({path: claim.written for path, claim in claims.items() if claim.written is not None}), + endpoints=MappingProxyType( + {path: claim.endpoint for path, claim in claims.items() if claim.endpoint is not None} + ), + ) + + +def read_configure_receipt(state_path: Path) -> ConfigureReceipt | None: + if not state_path.exists(): + return None try: - write_private_json(str(target), merged) + return ConfigureReceipt.model_validate_json(state_path.read_bytes()) + except (OSError, ValidationError) as e: + raise ClaudeSettingsError( + f"{state_path} is not a readable `lite configure claude` receipt ({e}). " + "Remove it and edit Claude Code's settings by hand if they still point at the proxy." + ) from e + + +def configure_claude_settings( + base_url: str, + credential: ClaudeCredential, + model: ModelChoice, + settings_path: Path, + state_path: Path, + owners: Sequence[SettingsFileOwner], + commit: Callable[[str, str], None] = commit_staged_json, +) -> None: + """Persistently route Claude Code through base_url, recording how to undo it. + + Both files are staged before either is committed, so a full disk or a read-only directory fails + before anything changes. The two commits are still two renames: a receipt rename that fails + discards the staged settings, and a settings rename that fails after the receipt landed puts the + earlier receipt back (or removes the new one), so the receipt on disk never describes settings + that were not written. `model`: StartOn pins the starting model, UnpinModel lets go of a pin an + earlier configure made (never of the user's own), KeepModel leaves it alone (a re-login). + """ + refuse_while_owned(settings_path, owners) + current: Final = load_json_or_empty(settings_path) + _env_object(current, settings_path) + earlier: Final = read_configure_receipt(state_path) + existing: Final = ( + _with(current, MODEL_KEY, earlier.previous[MODEL_KEY]) + if isinstance(model, UnpinModel) and earlier is not None and _ours(current, MODEL_KEY, earlier) + else current + ) + merged: Final = merge_claude_settings( + existing, base_url, credential, model.model if isinstance(model, StartOn) else None + ) + receipt: Final = _receipt(current, merged, earlier, settings_path.exists()) + target: Final = _write_target(settings_path) + try: + ensure_private_dir(state_path.parent) except OSError as e: - raise ClaudeSettingsError(f"Could not write {target}: {e}") from e + raise ClaudeSettingsError(f"Could not write {state_path}: {e}") from e + staged_receipt: Final = _stage(state_path, receipt.model_dump(mode="json")) + try: + staged_settings: Final = _stage(target, merged) + except ClaudeSettingsError: + discard_staged_json(staged_receipt) + raise + _land(state_path, staged_receipt, (staged_settings,), commit) + try: + _land(target, staged_settings, commit=commit) + except ClaudeSettingsError as settings_error: + try: + _land(state_path, None if earlier is None else _stage(state_path, earlier.model_dump(mode="json"))) + except ClaudeSettingsError as receipt_error: + raise ClaudeSettingsError( + f"{settings_error} The receipt at {state_path} now describes settings that were not written and " + f"could not be put back either ({receipt_error}); remove it before retrying." + ) from settings_error + raise + + +def _endpoint_text(endpoint: OwnedValue) -> str: + if not endpoint.present: + return f"no {ANTHROPIC_BASE_URL_KEY} (Anthropic's default endpoint)" + return endpoint.value if isinstance(endpoint.value, str) else json.dumps(endpoint.value) + + +def unconfigure_claude_settings( + settings_path: Path, state_path: Path, owners: Sequence[SettingsFileOwner] +) -> UnconfigureOutcome: + """Undo `lite configure claude`: put back every key still holding what configure wrote, leave the + rest alone, and withhold a credential the restored file would send to a different server than it + was issued for (the receipt stays, owning only those slots, so a later unconfigure can finish).""" + refuse_while_owned(settings_path, owners) + receipt: Final = read_configure_receipt(state_path) + if receipt is None: + raise ClaudeSettingsError( + f"Claude Code is not configured by `lite configure claude` (no receipt at {state_path}); nothing to undo." + ) + current: Final = load_json_or_empty(settings_path) + _env_object(current, settings_path) + ours: Final = tuple(path for path in receipt.written if _ours(current, path, receipt)) + kept: Final = tuple(path for path in receipt.written if path not in ours and _lookup(current, path).present) + put_back: Final = _with_all(current, MappingProxyType({path: receipt.previous[path] for path in ours})) + url_after: Final = _lookup(put_back, _BASE_URL_PATH) + withheld: Final = tuple( + WithheldCredential(path, _endpoint_text(receipt.endpoints[path])) + for path in _CREDENTIAL_PATHS + if path in ours and receipt.previous[path].present and receipt.endpoints[path] != url_after + ) + absent: Final = OwnedValue(present=False) + trimmed: Final = _with_all(put_back, MappingProxyType({item.key: absent for item in withheld})) + settings: Final = ( + trimmed + if _env(trimmed) or receipt.env_was_object + else _with_key(trimmed, ENV_KEY, OwnedValue(present=receipt.env_present, value=None)) + ) + target: Final = _write_target(settings_path) + file_removed: Final = not settings and not (receipt.file_existed and target.exists()) + kept_receipt: Final = ( # mutable-ok: pydantic serializes the update as given and rejects a mappingproxy + receipt.model_copy(update={"written": {item.key: _fingerprint(absent) for item in withheld}}) + if withheld + else None + ) + staged_settings: Final = None if file_removed else _stage(target, settings) + try: + staged_receipt: Final = ( + None if kept_receipt is None else _stage(state_path, kept_receipt.model_dump(mode="json")) + ) + except ClaudeSettingsError: + if staged_settings is not None: + discard_staged_json(staged_settings) + raise + _land(target, staged_settings, (staged_receipt,)) + _land(state_path, staged_receipt) + return UnconfigureOutcome( + restored=tuple(path for path in ours if _lookup(current, path) != _lookup(settings, path)), + kept=kept, + withheld=withheld, + file_removed=file_removed, + ) __all__ = ( "ANTHROPIC_API_KEY_KEY", + "ANTHROPIC_AUTH_TOKEN_KEY", "ANTHROPIC_BASE_URL_KEY", + "ANTHROPIC_DEFAULT_MODEL_ENV_KEYS", "API_KEY_HELPER_KEY", "AUTOROUTE_BACKUP_PATH", "BACKUP_PATH", "CLAUDE_SETTINGS_PATH", + "CONFIGURE_STATE_PATH", "ENABLE_GATEWAY_MODEL_DISCOVERY_KEY", "ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE", "ENABLE_TOOL_SEARCH_KEY", "ENABLE_TOOL_SEARCH_VALUE", "ENV_KEY", + "MODEL_KEY", + "OWNED_ENV_KEYS", + "OWNED_PATHS", + "OWNED_TOP_LEVEL_KEYS", "SETTINGS_FILE_OWNERS", + "STARTING_MODEL_ROLE", + "ApiKeyHelper", + "ClaudeCredential", "ClaudeSettingsError", + "ConfigureReceipt", + "KeepModel", + "ModelChoice", + "OwnedValue", "SettingsFileOwner", + "StartOn", + "StaticToken", + "UnconfigureOutcome", + "UnpinModel", + "WithheldCredential", + "configure_claude_settings", "load_json_or_empty", "merge_claude_settings", + "read_configure_receipt", + "refuse_while_owned", "resolve_api_key_helper", - "write_claude_settings", + "unconfigure_claude_settings", ) diff --git a/litellm/proxy/client/cli/commands/configure.py b/litellm/proxy/client/cli/commands/configure.py new file mode 100644 index 00000000000..f3a98a48476 --- /dev/null +++ b/litellm/proxy/client/cli/commands/configure.py @@ -0,0 +1,252 @@ +"""`lite configure claude` and `lite unconfigure claude`: persistent Claude Code wiring, undoable.""" + +import re +import sys +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import Final + +import click +from InquirerPy import inquirer +from InquirerPy.base.control import Choice + +from .auth import CliContextObj, context_secret_vault, get_stored_api_key +from .claude_settings import ( + CLAUDE_SETTINGS_PATH, + CONFIGURE_STATE_PATH, + SETTINGS_FILE_OWNERS, + STARTING_MODEL_ROLE, + ApiKeyHelper, + ClaudeCredential, + ClaudeSettingsError, + ModelChoice, + StartOn, + StaticToken, + UnconfigureOutcome, + UnpinModel, + configure_claude_settings, + refuse_while_owned, + resolve_api_key_helper, + unconfigure_claude_settings, +) +from .pi import ListingFailure, PiSyncError, fetch_model_ids +from .up import ensure_fresh_login + +_LISTED_MODELS_SHOWN: Final = 20 +_CLAUDE_TARGET: Final = "claude" +_TARGETS: Final = ((_CLAUDE_TARGET, "Claude Code (CLI)"),) +_KEEP_DEFAULT_MODEL: Final = "Keep Claude Code's own default" +_CLAUDE_CODE_PICKER_FILTER: Final = re.compile(r"claude|anthropic", re.IGNORECASE) +_MODEL_OPTION_HELP: Final = ( + f"Proxy model to set as {STARTING_MODEL_ROLE}. Must be listed on /v1/models for the key; without it, " + "Claude Code keeps its own default and a pin an earlier configure made is let go of. Nothing pins Claude " + "Code's sub-agent or background tiers; `lite autoroute up` is the mode that does." +) + + +def resolve_credential(ctx: click.Context, api_key: str | None) -> tuple[ClaudeCredential, str]: + """The credential to write and the key to check the proxy with. + + An explicit key (--api-key, `lite --api-key`, LITELLM_PROXY_API_KEY) is long-lived and goes + into settings.json as a static token. Without one, the stored `lite login` credential is used + the way `lite login --config-claude` uses it, through apiKeyHelper, since it expires within a + day and renews in place there; a missing or stale login is refreshed first, as `lite up` does. + """ + ctx_obj: Final[CliContextObj] = ctx.obj + explicit: Final = api_key or (None if ctx_obj.get("api_key_from_token_file") else ctx_obj.get("api_key")) + if explicit: + return StaticToken(explicit), explicit + base_url: Final = ctx_obj["base_url"] + ensure_fresh_login(ctx) + stored: Final = get_stored_api_key(expected_base_url=base_url, vault=context_secret_vault(ctx)) + if not stored: + raise ClaudeSettingsError("Login did not produce a usable token.") + return ApiKeyHelper(resolve_api_key_helper(base_url)), stored + + +def _start(ctx: click.Context, api_key: str | None) -> tuple[ClaudeCredential, tuple[str, ...]]: + """Every configure path begins the same way: the local ownership check first, so a `lite up` + session is refused before any login prompt or request, then the credential, then the listing.""" + try: + refuse_while_owned(CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS) + credential, key = resolve_credential(ctx, api_key) + except ClaudeSettingsError as e: + raise click.ClickException(str(e)) + return credential, _listed_models(ctx.obj["base_url"], key) + + +def _listing_error(base_url: str, error: PiSyncError) -> str: + """The hint that fits how the listing failed: only an unreachable proxy gets the "is it running" question.""" + if error.kind is ListingFailure.REJECTED: + return f"LiteLLM rejected your key (HTTP {error.status}). Run `lite login` to refresh it, or pass a valid --api-key." + if error.kind is ListingFailure.UNREACHABLE: + return f"{error.message} Is the proxy at {base_url} running, and is --base-url (or LITELLM_PROXY_URL) correct?" + if error.kind is ListingFailure.EMPTY: + return f"{error.message} Claude Code would have nothing to run; give the key access to at least one model." + return f"{error.message} The proxy at {base_url} answered, so check that it is a LiteLLM proxy and is healthy." + + +def _listed_models(base_url: str, key: str) -> tuple[str, ...]: + listed: Final = fetch_model_ids(base_url, key) + if isinstance(listed, PiSyncError): + raise click.ClickException(_listing_error(base_url, listed)) + return listed + + +def _model_choice(model: str | None) -> ModelChoice: + return StartOn(model) if model is not None else UnpinModel() + + +def _apply_claude(ctx: click.Context, credential: ClaudeCredential, listed: Sequence[str], model: str | None) -> None: + ctx_obj: Final[CliContextObj] = ctx.obj + base_url: Final = ctx_obj["base_url"] + if model is not None and model not in listed: + shown: Final = ", ".join(listed[:_LISTED_MODELS_SHOWN]) + more: Final = f", and {len(listed) - _LISTED_MODELS_SHOWN} more" if len(listed) > _LISTED_MODELS_SHOWN else "" + raise click.ClickException( + f"{model!r} is not served by {base_url} for this key. /v1/models lists: {shown}{more}." + ) + try: + configure_claude_settings( + base_url, credential, _model_choice(model), CLAUDE_SETTINGS_PATH, CONFIGURE_STATE_PATH, SETTINGS_FILE_OWNERS + ) + except ClaudeSettingsError as e: + raise click.ClickException(str(e)) + in_picker: Final = sum(1 for listed_model in listed if _CLAUDE_CODE_PICKER_FILTER.search(listed_model)) + click.echo(f"Configured Claude Code: {CLAUDE_SETTINGS_PATH} now routes through {base_url}.") + click.echo( + "Credential: your virtual key, stored in the file as ANTHROPIC_AUTH_TOKEN." + if isinstance(credential, StaticToken) + else "Credential: your `lite login`, read through apiKeyHelper on every request, so a later login renews it." + ) + click.echo( + f"Starting model: {model} ({STARTING_MODEL_ROLE}); switch any time with /model." + if model is not None + else "Starting model: not pinned (Claude Code's default, or a model you set yourself); switch with /model, or " + "pass --model to start on a proxy model." + ) + click.echo( + f"/model will list {in_picker} of the proxy's {len(listed)} models (Claude Code shows only ids containing " + "'claude' or 'anthropic')." + ) + click.echo("Start `claude` from any terminal. Undo with `lite unconfigure claude`.") + if isinstance(credential, StaticToken) and CLAUDE_SETTINGS_PATH.is_symlink(): + click.echo( + f"Note: {CLAUDE_SETTINGS_PATH} is a symlink to {CLAUDE_SETTINGS_PATH.resolve()}, so your key now lives in " + "that file; keep it out of version control.", + err=True, + ) + + +def _pick_targets() -> tuple[str, ...]: + picked: Final = inquirer.checkbox( + message="Which agents should route through LiteLLM?", + choices=[Choice(value, name=label, enabled=True) for value, label in _TARGETS], + validate=lambda chosen: len(chosen) > 0, + invalid_message="Pick at least one.", + ).execute() + return tuple(str(value) for value in picked) + + +def _pick_model(listed: Sequence[str]) -> str | None: + picked: Final = inquirer.fuzzy( + message="Model Claude Code starts on (type to filter; /model switches any time):", + choices=[_KEEP_DEFAULT_MODEL, *listed], + ).execute() + return None if picked == _KEEP_DEFAULT_MODEL else str(picked) + + +def interactive_configure( + ctx: click.Context, + pick_targets: Callable[[], tuple[str, ...]] = _pick_targets, + pick_model: Callable[[Sequence[str]], str | None] = _pick_model, +) -> None: + """`lite configure` with no agent named: ask which agents to wire and which model to pin.""" + targets: Final = pick_targets() + if _CLAUDE_TARGET not in targets: + return + credential, listed = _start(ctx, None) + _apply_claude(ctx, credential, listed, pick_model(listed)) + + +@click.group(name="configure", invoke_without_command=True) +@click.pass_context +def configure_group(ctx: click.Context) -> None: + """Persistently route a coding agent through your LiteLLM proxy. + + With no agent named, asks which agents to wire and which proxy model to pin. + """ + if ctx.invoked_subcommand is not None: + return + if not sys.stdin.isatty(): + raise click.ClickException( + "`lite configure` asks questions, so it needs a terminal. Non-interactively, run " + "`lite configure claude --api-key --model `." + ) + interactive_configure(ctx) + + +@click.group(name="unconfigure") +def unconfigure_group() -> None: + """Undo `lite configure` for a coding agent.""" + + +@configure_group.command(name="claude") +@click.option( + "--api-key", + "api_key", + default=None, + help="Long-lived LiteLLM virtual key written into Claude Code's settings. Defaults to the `lite --api-key` / " + "LITELLM_PROXY_API_KEY value; with neither, your `lite login` credential is used through apiKeyHelper.", +) +@click.option("--model", default=None, help=_MODEL_OPTION_HELP) +@click.pass_context +def configure_claude(ctx: click.Context, api_key: str | None, model: str | None) -> None: + """Route every Claude Code session through your LiteLLM proxy until `lite unconfigure claude`. + + Patches ~/.claude/settings.json in place: the proxy URL, your credential (a virtual key as a + static token, or your `lite login` through apiKeyHelper), and gateway model discovery so + /model lists the proxy's models; --model picks the one Claude Code starts on. Every other + setting is kept, and what changed is recorded so `lite unconfigure claude` can put it back. + Assumes the proxy is already running. + """ + credential, listed = _start(ctx, api_key) + _apply_claude(ctx, credential, listed, model) + + +@unconfigure_group.command(name="claude") +def unconfigure_claude() -> None: + """Return Claude Code's settings to what they were before `lite configure claude`. + + Also undoes `lite login --config-claude`. Only keys still holding what configure wrote are + put back; anything you changed since is left as it is and named in the output. + """ + try: + outcome: Final = unconfigure_claude_settings(CLAUDE_SETTINGS_PATH, CONFIGURE_STATE_PATH, SETTINGS_FILE_OWNERS) + except ClaudeSettingsError as e: + raise click.ClickException(str(e)) + _report_unconfigure(CLAUDE_SETTINGS_PATH, CONFIGURE_STATE_PATH, outcome) + + +def _report_unconfigure(settings_path: Path, state_path: Path, outcome: UnconfigureOutcome) -> None: + """Say what unconfigure did, naming only keys whose value it changed.""" + if outcome.file_removed: + click.echo( + f"No settings file remains at {settings_path}; it held nothing but `lite configure claude`'s own keys." + ) + elif outcome.restored: + click.echo(f"Restored in {settings_path}: {', '.join(outcome.restored)}.") + else: + click.echo(f"Nothing in {settings_path} was still ours to restore.") + if outcome.kept: + click.echo(f"Left as you changed them since: {', '.join(outcome.kept)}.") + if outcome.withheld: + click.echo( + "Left removed, since the file now points at a different server than they were issued for: " + + "; ".join(f"{item.key} (captured with {item.endpoint})" for item in outcome.withheld) + + f". They stay in {state_path}: point env.ANTHROPIC_BASE_URL back and run `lite unconfigure claude` " + "again to put them back, or delete that file to drop them." + ) + + +__all__ = ("configure_group", "interactive_configure", "resolve_credential", "unconfigure_group") diff --git a/litellm/proxy/client/cli/commands/pi.py b/litellm/proxy/client/cli/commands/pi.py index 7b0c1970c4e..70c89a853e3 100644 --- a/litellm/proxy/client/cli/commands/pi.py +++ b/litellm/proxy/client/cli/commands/pi.py @@ -10,6 +10,7 @@ import os import tempfile from collections.abc import Callable, Mapping from dataclasses import dataclass +from enum import StrEnum from pathlib import Path from types import MappingProxyType from typing import Final @@ -20,11 +21,28 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError PI_CONFIG_DIR_ENV: Final = "PI_CODING_AGENT_DIR" PI_PROVIDER_NAME: Final = "litellm" LITELLM_PROXY_API_KEY_ENV: Final = "LITELLM_PROXY_API_KEY" +_REJECTED_STATUSES: Final = frozenset((401, 403)) + + +class ListingFailure(StrEnum): + """Why a proxy could not be listed, decided once where the HTTP outcome is classified. + + `unreachable` means no response at all; the other kinds prove the proxy answered, so callers + must not suggest checking whether it is running. + """ + + UNREACHABLE = "unreachable" + REJECTED = "rejected" + BAD_BODY = "bad_body" + EMPTY = "empty" + OTHER = "other" @dataclass(frozen=True, slots=True) class PiSyncError: message: str + status: int | None = None + kind: ListingFailure | None = None @dataclass(frozen=True, slots=True) @@ -65,16 +83,20 @@ def fetch_model_ids( timeout=10, ) except requests.RequestException as e: - return PiSyncError(f"Could not list models from the proxy: {e}") + return PiSyncError(f"Could not list models from the proxy: {e}", kind=ListingFailure.UNREACHABLE) if resp.status_code != 200: - return PiSyncError(f"The proxy returned HTTP {resp.status_code} for /v1/models; cannot build pi's model list.") + return PiSyncError( + f"The proxy returned HTTP {resp.status_code} for /v1/models; cannot list models.", + resp.status_code, + ListingFailure.REJECTED if resp.status_code in _REJECTED_STATUSES else ListingFailure.OTHER, + ) try: listing: Final = _ModelList.model_validate(resp.json()) except (ValueError, ValidationError) as e: - return PiSyncError(f"Unexpected /v1/models response from the proxy: {e}") + return PiSyncError(f"Unexpected /v1/models response from the proxy: {e}", kind=ListingFailure.BAD_BODY) ids: Final = tuple(dict.fromkeys(model.id for model in listing.data)) if not ids: - return PiSyncError("The proxy returned no models for your key, so pi would have nothing to run.") + return PiSyncError("The proxy returned no models for your key.", kind=ListingFailure.EMPTY) return ids @@ -200,6 +222,7 @@ __all__ = ( "LITELLM_PROXY_API_KEY_ENV", "PI_CONFIG_DIR_ENV", "PI_PROVIDER_NAME", + "ListingFailure", "ModelLimits", "PiSyncError", "fetch_model_ids", diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index b7c02866d6f..ffece87ab83 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -23,6 +23,7 @@ from .auth import CliContextObj, context_secret_vault, get_stored_api_key, load_ from .claude_settings import ( BACKUP_PATH, CLAUDE_SETTINGS_PATH, + ApiKeyHelper, ClaudeSettingsError, load_json_or_empty, merge_claude_settings, @@ -123,7 +124,7 @@ def _stored_login_is_pkce(vault: SecretVault) -> bool: return token_data is not None and token_data.get("refresh_token") is not None -def _ensure_fresh_login(ctx: click.Context) -> None: +def ensure_fresh_login(ctx: click.Context) -> None: ctx_obj: Final[CliContextObj] = ctx.obj base_url: Final = ctx_obj["base_url"].rstrip("/") vault: Final = context_secret_vault(ctx) @@ -141,7 +142,7 @@ def _ensure_fresh_login(ctx: click.Context) -> None: click.echo("No fresh LiteLLM login found for this proxy; starting login...") ctx.invoke(login, pkce=pkce) if not _usable_login(get_stored_api_key(expected_base_url=base_url, vault=vault), vault): - raise UpError("Login did not produce a usable token; cannot start `lite up`.") + raise UpError("Login did not produce a usable token.") def _restore_and_report() -> None: @@ -169,7 +170,7 @@ def up(ctx: click.Context) -> None: base_url: Final = ctx.obj["base_url"] try: - _ensure_fresh_login(ctx) + ensure_fresh_login(ctx) api_key: Final = resolve_api_key(ctx) verify_proxy_key(base_url, api_key) @@ -190,7 +191,7 @@ def up(ctx: click.Context) -> None: ) CLAUDE_SETTINGS_PATH.parent.mkdir(exist_ok=True) - merged: Final = merge_claude_settings(original_settings, base_url, api_key_helper) + merged: Final = merge_claude_settings(original_settings, base_url, ApiKeyHelper(api_key_helper)) with open(CLAUDE_SETTINGS_PATH, "w") as f: json.dump(merged, f, indent=2) except (AgentRunError, ClaudeSettingsError) as e: diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index eae1b0f5bc9..b0e81a222c0 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -13,6 +13,7 @@ from .commands.auth import auth_group, context_secret_vault, get_stored_api_key, from .commands.autoroute.commands import autoroute_group from .commands.chat import chat from .commands.config import config_commands, get_config_value, hidden_command_names +from .commands.configure import configure_group, unconfigure_group from .commands.credentials import credentials from .commands.debug import debug from .commands.encryption import encryption @@ -162,6 +163,9 @@ cli.add_command(model_groups) # Add the autoroute command group (QA auto-routing against your real proxy) cli.add_command(autoroute_group, name="autoroute") cli.add_command(config_commands) +# Add configure/unconfigure (persistently wire a coding agent to the proxy with a virtual key) +cli.add_command(configure_group) +cli.add_command(unconfigure_group) if __name__ == "__main__": diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py index 4a3b3ef22c4..77742ea9f9f 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py @@ -159,6 +159,10 @@ class TestUpCommand: assert captured["settings"]["env"]["ANTHROPIC_AUTH_TOKEN"] == "fixed-master-key" assert captured["settings"]["env"]["ENABLE_TOOL_SEARCH"] == "true" assert "apiKeyHelper" not in captured["settings"] + # The ephemeral proxy serves only the autorouter, so a starting model left by + # `lite configure claude --model` or a user pin would 400 on the first message. + assert captured["settings"]["model"] == "autorouter" + assert captured["settings"]["env"]["ANTHROPIC_DEFAULT_SONNET_MODEL"] == "autorouter" assert captured["settings_mode"] == 0o600 assert terminate_calls == [99999] diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py b/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py deleted file mode 100644 index 87a33c79a79..00000000000 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py +++ /dev/null @@ -1,63 +0,0 @@ -from litellm.proxy.client.cli.commands.autoroute.settings import ( - ANTHROPIC_DEFAULT_MODEL_ENV_KEYS, - merge_claude_settings_static_token, -) - - -def test_preserves_unrelated_top_level_keys(): - merged = merge_claude_settings_static_token({"theme": "dark"}, "http://127.0.0.1:4000", "token-abc") - assert merged["theme"] == "dark" - - -def test_preserves_unrelated_env_keys(): - settings = {"env": {"SOME_OTHER_VAR": "value"}} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert merged["env"]["SOME_OTHER_VAR"] == "value" - - -def test_sets_base_url_and_auth_token(): - merged = merge_claude_settings_static_token({}, "http://127.0.0.1:4000/", "token-abc") - assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:4000" - assert merged["env"]["ANTHROPIC_AUTH_TOKEN"] == "token-abc" - assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true" - - -def test_preserves_existing_tool_search(): - settings = {"env": {"ENABLE_TOOL_SEARCH": "false"}} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false" - - -def test_drops_stray_api_key(): - settings = {"env": {"ANTHROPIC_API_KEY": "leaked-key"}} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert "ANTHROPIC_API_KEY" not in merged["env"] - - -def test_removes_existing_api_key_helper(): - settings = {"apiKeyHelper": "/usr/local/bin/lite auth print-token"} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert "apiKeyHelper" not in merged - - -def test_does_not_mutate_input(): - settings = {"env": {"FOO": "bar"}, "apiKeyHelper": "old-helper"} - merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert settings == {"env": {"FOO": "bar"}, "apiKeyHelper": "old-helper"} - - -def test_forces_all_claude_code_default_model_tiers_to_the_autorouter(): - # A bare "*" model_name deployment looks like the obvious way to catch every request - # regardless of which model Claude Code thinks it's using, but Router's auto-router - # registry is keyed by the literal requested model string with no wildcard resolution - # (litellm/router.py:10711-10717) -- so the only reliable way to make every one of Claude - # Code's own tiers hit the auto-router is to override the env vars it reads per tier. - merged = merge_claude_settings_static_token({}, "http://127.0.0.1:4000", "token-abc") - for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS: - assert merged["env"][key] == "autorouter" - - -def test_overrides_a_preexisting_default_model_env_var(): - settings = {"env": {"ANTHROPIC_DEFAULT_SONNET_MODEL": "claude-opus-4-8"}} - merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") - assert merged["env"]["ANTHROPIC_DEFAULT_SONNET_MODEL"] == "autorouter" diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 821323e722c..7d5cb61c062 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -27,6 +27,7 @@ from litellm.proxy.client.cli.commands.auth import ( print_token, whoami, ) +from litellm.proxy.client.cli.commands import auth as auth_module from litellm.proxy.client.cli.commands.claude_settings import SettingsFileOwner @@ -1398,8 +1399,10 @@ class TestLoginConfigClaude: def setup_method(self): self.runner = CliRunner() - def _run_login(self, tmp_path, args, base_url="https://test.example.com"): + def _run_login(self, tmp_path, monkeypatch, args, base_url="https://test.example.com"): settings_path = tmp_path / "claude" / "settings.json" + monkeypatch.setattr(auth_module, "CLAUDE_SETTINGS_PATH", settings_path) + monkeypatch.setattr(auth_module, "CONFIGURE_STATE_PATH", tmp_path / "claude_configure_state.json") backup_path = tmp_path / "claude_settings_backup.json" poll_response = Mock() poll_response.status_code = 200 @@ -1416,7 +1419,6 @@ class TestLoginConfigClaude: patch("requests.get", return_value=poll_response), patch("litellm.proxy.client.cli.commands.auth.save_cli_token"), patch("litellm.proxy.client.cli.interface.show_commands"), - patch("litellm.proxy.client.cli.commands.auth.CLAUDE_SETTINGS_PATH", settings_path), patch( "litellm.proxy.client.cli.commands.auth.SETTINGS_FILE_OWNERS", (SettingsFileOwner(backup_path, "lite up", "lite down"),), @@ -1429,16 +1431,16 @@ class TestLoginConfigClaude: result = self.runner.invoke(login, args, obj={"base_url": base_url}) return result, settings_path, backup_path - def test_default_login_does_not_touch_claude_settings(self, tmp_path): - result, settings_path, _backup_path = self._run_login(tmp_path, []) + def test_default_login_does_not_touch_claude_settings(self, tmp_path, monkeypatch): + result, settings_path, _backup_path = self._run_login(tmp_path, monkeypatch, []) assert result.exit_code == 0 assert "Login successful!" in result.output assert not settings_path.exists() assert "Configured Claude Code" not in result.output - def test_flag_writes_the_settings_file_and_reports_success(self, tmp_path): - result, settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + def test_flag_writes_the_settings_file_and_reports_success(self, tmp_path, monkeypatch): + result, settings_path, _backup_path = self._run_login(tmp_path, monkeypatch, ["--config-claude"]) assert result.exit_code == 0 written = json.loads(settings_path.read_text()) @@ -1446,25 +1448,43 @@ class TestLoginConfigClaude: assert written["env"]["ENABLE_TOOL_SEARCH"] == "true" assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://test.example.com auth print-token" assert "Configured Claude Code" in result.output + assert "pins a proxy model for every tier" not in result.output + assert "the model Claude Code starts on" in result.output - def test_flag_preserves_unrelated_settings_on_an_existing_file(self, tmp_path): + def test_flag_preserves_unrelated_settings_on_an_existing_file(self, tmp_path, monkeypatch): settings_path = tmp_path / "claude" / "settings.json" settings_path.parent.mkdir(parents=True) settings_path.write_text(json.dumps({"theme": "dark", "env": {"KEEP": "me"}})) - result, _settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + result, _settings_path, _backup_path = self._run_login(tmp_path, monkeypatch, ["--config-claude"]) assert result.exit_code == 0 written = json.loads(settings_path.read_text()) assert written["theme"] == "dark" assert written["env"]["KEEP"] == "me" - def test_settings_failure_is_reported_without_claiming_login_failed(self, tmp_path): + def test_refuses_before_logging_in_while_lite_up_holds_the_settings(self, tmp_path, monkeypatch): + # The local precondition comes first: no browser, no token stored, no "Login successful!". + backup_path = tmp_path / "claude_settings_backup.json" + backup_path.write_text("{}") + monkeypatch.setattr(auth_module, "CLAUDE_SETTINGS_PATH", tmp_path / "claude" / "settings.json") + monkeypatch.setattr( + auth_module, "SETTINGS_FILE_OWNERS", (SettingsFileOwner(backup_path, "lite up", "lite down"),) + ) + with patch("requests.post") as post, patch("webbrowser.open") as browser: + result = self.runner.invoke(login, ["--config-claude"], obj={"base_url": "https://test.example.com"}) + assert result.exit_code != 0 + assert "not logging in" in result.output and "lite down" in result.output + assert "Login successful!" not in result.output + post.assert_not_called() + browser.assert_not_called() + + def test_settings_failure_is_reported_without_claiming_login_failed(self, tmp_path, monkeypatch): settings_path = tmp_path / "claude" / "settings.json" settings_path.parent.mkdir(parents=True) settings_path.write_text("not json at all {{{") - result, _settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + result, _settings_path, _backup_path = self._run_login(tmp_path, monkeypatch, ["--config-claude"]) assert result.exit_code != 0 assert "Login successful!" in result.output diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py index e5f2a9d95bd..9072fe26b00 100644 --- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py +++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py @@ -1,4 +1,5 @@ import json +import os import shlex import stat import time @@ -9,14 +10,25 @@ from click.testing import CliRunner from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord from litellm.proxy.client.cli import cli +from litellm.litellm_core_utils.private_json import commit_staged_json from litellm.proxy.client.cli.commands.claude_settings import ( + ANTHROPIC_DEFAULT_MODEL_ENV_KEYS, AUTOROUTE_BACKUP_PATH, BACKUP_PATH, + OWNED_ENV_KEYS, + OWNED_TOP_LEVEL_KEYS, SETTINGS_FILE_OWNERS, + ApiKeyHelper, ClaudeSettingsError, + KeepModel, SettingsFileOwner, + StartOn, + StaticToken, + UnpinModel, + configure_claude_settings, + merge_claude_settings, resolve_api_key_helper, - write_claude_settings, + unconfigure_claude_settings, ) @@ -24,6 +36,7 @@ def _owners(*backup_paths): """Stand-in owners for the real `lite up` / `lite autoroute up` registry.""" return tuple(SettingsFileOwner(path, "lite up", "lite down") for path in backup_paths) + CLAUDE_SETTINGS_MODULE = "litellm.proxy.client.cli.commands.claude_settings" AUTH_MODULE = "litellm.proxy.client.cli.commands.auth" WINDOWS_LITE_EXE = "C:\\Users\\u\\AppData\\Local\\Programs\\Python\\Python313\\Scripts\\lite.EXE" @@ -97,17 +110,28 @@ def lite_on_path(): yield -class TestWriteClaudeSettings: +def _helper_configure(base_url, settings_path, owners, state_path=None): + """`lite login --config-claude`'s shape: the login credential behind apiKeyHelper, no pinned model.""" + state = state_path if state_path is not None else settings_path.parent.parent / "state.json" + root = base_url.rstrip("/") + configure_claude_settings( + root, ApiKeyHelper(resolve_api_key_helper(root)), KeepModel(), settings_path, state, owners + ) + + +class TestConfigureWithTheLoginHelper: def test_creates_the_file_and_its_parent_when_missing(self, paths, lite_on_path): settings_path, backup_path = paths assert not settings_path.parent.exists() - write_claude_settings("https://proxy.example.com/", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com/", settings_path, _owners(backup_path)) written = json.loads(settings_path.read_text()) assert written["env"]["ANTHROPIC_BASE_URL"] == "https://proxy.example.com" assert written["env"]["ENABLE_TOOL_SEARCH"] == "true" + assert written["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1" assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://proxy.example.com auth print-token" + assert "model" not in written def test_updates_an_existing_file_preserving_unrelated_settings(self, paths, lite_on_path): settings_path, backup_path = paths @@ -123,7 +147,7 @@ class TestWriteClaudeSettings: ) ) - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) written = json.loads(settings_path.read_text()) assert written["theme"] == "dark" @@ -135,26 +159,31 @@ class TestWriteClaudeSettings: def test_rerunning_against_a_new_proxy_refreshes_both_base_url_and_helper(self, paths, lite_on_path): settings_path, backup_path = paths - write_claude_settings("https://first.example.com", settings_path, _owners(backup_path)) - write_claude_settings("https://second.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://first.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://second.example.com", settings_path, _owners(backup_path)) written = json.loads(settings_path.read_text()) assert written["env"]["ANTHROPIC_BASE_URL"] == "https://second.example.com" assert "second.example.com" in written["apiKeyHelper"] assert "first.example.com" not in written["apiKeyHelper"] - def test_drops_a_stray_static_api_key_so_the_helper_token_wins(self, paths, lite_on_path): + def test_drops_stray_static_credentials_so_the_helper_token_wins(self, paths, lite_on_path): + # Claude Code prefers ANTHROPIC_AUTH_TOKEN over apiKeyHelper, so a virtual key left behind + # by an earlier `lite configure claude --api-key` would silently keep winning. settings_path, backup_path = paths settings_path.parent.mkdir(parents=True) - settings_path.write_text(json.dumps({"env": {"ANTHROPIC_API_KEY": "sk-leaked"}})) + settings_path.write_text( + json.dumps({"env": {"ANTHROPIC_API_KEY": "sk-leaked", "ANTHROPIC_AUTH_TOKEN": "sk-old"}}) + ) - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) - assert "ANTHROPIC_API_KEY" not in json.loads(settings_path.read_text())["env"] + env = json.loads(settings_path.read_text())["env"] + assert "ANTHROPIC_API_KEY" not in env and "ANTHROPIC_AUTH_TOKEN" not in env def test_written_file_is_owner_only(self, paths, lite_on_path): settings_path, backup_path = paths - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert stat.S_IMODE(settings_path.stat().st_mode) == 0o600 def test_refuses_while_lite_up_holds_a_backup(self, paths, lite_on_path): @@ -162,7 +191,7 @@ class TestWriteClaudeSettings: backup_path.write_text("{}") with pytest.raises(ClaudeSettingsError, match="lite down"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert not settings_path.exists() @@ -172,7 +201,7 @@ class TestWriteClaudeSettings: settings_path.write_text("not json at all {{{") with pytest.raises(ClaudeSettingsError, match="invalid JSON"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert settings_path.read_text() == "not json at all {{{" @@ -180,7 +209,7 @@ class TestWriteClaudeSettings: settings_path, backup_path = paths with patch(f"{CLAUDE_SETTINGS_MODULE}.shutil.which", return_value=None): with pytest.raises(ClaudeSettingsError, match="Could not find `lite`"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert not settings_path.exists() @@ -196,7 +225,7 @@ class TestWriteClaudeSettings: settings_path.write_bytes(b'{"theme": "\xff\xfe"}') with pytest.raises(ClaudeSettingsError, match="invalid JSON"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) def test_reports_an_actionable_error_when_the_file_cannot_be_read(self, paths, lite_on_path): """An unreadable settings file must not surface as "Authentication failed". @@ -210,16 +239,18 @@ class TestWriteClaudeSettings: settings_path.mkdir() with pytest.raises(ClaudeSettingsError, match="Could not read"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) def test_reports_an_actionable_error_when_the_file_cannot_be_written(self, paths, lite_on_path): settings_path, backup_path = paths - with patch( - f"{CLAUDE_SETTINGS_MODULE}.write_private_json", - side_effect=OSError("Read-only file system"), - ): - with pytest.raises(ClaudeSettingsError, match="Read-only file system"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + settings_path.parent.mkdir(parents=True) + settings_path.parent.chmod(0o500) + try: + with pytest.raises(ClaudeSettingsError, match="Could not write"): + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) + finally: + settings_path.parent.chmod(0o700) + assert not settings_path.exists() class TestApiKeyHelperIsActuallyInvocable: @@ -303,7 +334,7 @@ class TestConflictingOwnersOfTheSettingsFile: backup.write_text("{}") stand_in = SettingsFileOwner(backup, owner.start_command, owner.stop_command) with pytest.raises(ClaudeSettingsError, match="currently managing"): - write_claude_settings("https://proxy.example.com", settings_path, (stand_in,)) + _helper_configure("https://proxy.example.com", settings_path, (stand_in,)) backup.unlink() assert not settings_path.exists() @@ -314,9 +345,9 @@ class TestConflictingOwnersOfTheSettingsFile: autoroute = SettingsFileOwner(backup, "lite autoroute up", "lite autoroute down") with pytest.raises(ClaudeSettingsError, match="`lite autoroute up` is currently managing"): - write_claude_settings("https://proxy.example.com", settings_path, (autoroute,)) + _helper_configure("https://proxy.example.com", settings_path, (autoroute,)) with pytest.raises(ClaudeSettingsError, match="Run `lite autoroute down` first"): - write_claude_settings("https://proxy.example.com", settings_path, (autoroute,)) + _helper_configure("https://proxy.example.com", settings_path, (autoroute,)) def test_the_registry_matches_the_paths_the_commands_actually_use(self): """A second definition of the autoroute dir must not drift from this one.""" @@ -341,7 +372,7 @@ class TestDoesNotDestroyUserOwnedStructure: link.parent.mkdir() link.symlink_to(real) - write_claude_settings("https://proxy.example.com", link, ()) + _helper_configure("https://proxy.example.com", link, ()) assert link.is_symlink() assert json.loads(real.read_text())["env"]["ANTHROPIC_BASE_URL"] == "https://proxy.example.com" @@ -354,6 +385,456 @@ class TestDoesNotDestroyUserOwnedStructure: settings_path.write_text(json.dumps({"theme": "dark", "env": "not-an-object"})) with pytest.raises(ClaudeSettingsError, match="non-object"): - write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + _helper_configure("https://proxy.example.com", settings_path, _owners(backup_path)) assert json.loads(settings_path.read_text())["env"] == "not-an-object" + + +class TestMergeClaudeSettings: + """One merge for every way Claude Code gets wired: `lite up`, `lite login --config-claude`, + `lite configure claude` and `lite autoroute up`.""" + + def test_a_static_token_lands_in_env_and_the_helper_slot_is_cleared(self): + settings = {"apiKeyHelper": "/usr/local/bin/lite auth print-token", "env": {"ANTHROPIC_API_KEY": "leaked"}} + merged = merge_claude_settings(settings, "http://127.0.0.1:4000/", StaticToken("token-abc")) + assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:4000" + assert merged["env"]["ANTHROPIC_AUTH_TOKEN"] == "token-abc" + assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true" + assert merged["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1" + assert "ANTHROPIC_API_KEY" not in merged["env"] + assert "apiKeyHelper" not in merged + assert "model" not in merged + assert not any(key in merged["env"] for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS) + + def test_a_helper_lands_top_level_and_the_static_slots_are_cleared(self): + settings = {"env": {"ANTHROPIC_AUTH_TOKEN": "sk-old", "ANTHROPIC_API_KEY": "leaked"}} + merged = merge_claude_settings(settings, "http://127.0.0.1:4000", ApiKeyHelper("lite auth print-token")) + assert merged["apiKeyHelper"] == "lite auth print-token" + assert "ANTHROPIC_AUTH_TOKEN" not in merged["env"] and "ANTHROPIC_API_KEY" not in merged["env"] + + def test_keeps_existing_switch_values_and_unrelated_keys_without_mutating_the_input(self): + settings = {"theme": "dark", "env": {"SOME_OTHER_VAR": "value", "ENABLE_TOOL_SEARCH": "false"}} + merged = merge_claude_settings(settings, "http://127.0.0.1:4000", StaticToken("token-abc")) + assert merged["theme"] == "dark" + assert merged["env"]["SOME_OTHER_VAR"] == "value" + assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false" + assert settings == {"theme": "dark", "env": {"SOME_OTHER_VAR": "value", "ENABLE_TOOL_SEARCH": "false"}} + + def test_a_default_model_sets_only_the_row_claude_code_starts_on(self): + merged = merge_claude_settings( + {}, "http://127.0.0.1:4000", StaticToken("token-abc"), default_model="claude-auto" + ) + assert merged["model"] == "claude-auto" + assert not any(key in merged["env"] for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS) + + def test_a_tier_model_forces_every_claude_code_tier_as_autoroute_needs(self): + # Router's auto-router registry is keyed by the literal requested model string with no + # wildcard resolution, so `lite autoroute up` overrides the env var each tier reads. + settings = {"env": {"ANTHROPIC_DEFAULT_SONNET_MODEL": "claude-opus-4-8"}} + merged = merge_claude_settings( + settings, "http://127.0.0.1:4000", StaticToken("token-abc"), tier_model="autorouter" + ) + assert {merged["env"][key] for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS} == {"autorouter"} + assert "model" not in merged + + def test_touches_exactly_the_declared_owned_keys(self): + # The receipt and unconfigure restore exactly OWNED_*_KEYS, so a key the merge writes outside + # that table would be written by configure and never undone. + settings = { + "theme": "dark", + "permissions": {"allow": ["Bash"]}, + "env": {"KEEP_ME": "1", "ANTHROPIC_API_KEY": "old", "ENABLE_TOOL_SEARCH": "false"}, + "apiKeyHelper": "old-helper", + "model": "old-model", + } + for credential in (StaticToken("token-abc"), ApiKeyHelper("helper")): + merged = merge_claude_settings(settings, "http://127.0.0.1:4000", credential, default_model="claude-auto") + changed_top_level = {key for key in set(settings) | set(merged) if settings.get(key) != merged.get(key)} + assert changed_top_level - {"env"} <= set(OWNED_TOP_LEVEL_KEYS) + changed_env = { + key + for key in set(settings["env"]) | set(merged["env"]) + if settings["env"].get(key) != merged["env"].get(key) + } + assert changed_env <= set(OWNED_ENV_KEYS) + assert merged["permissions"] == {"allow": ["Bash"]} + assert merged["env"]["KEEP_ME"] == "1" + + +PROXY = "http://127.0.0.1:4000" +ANTHROPIC = "https://api.anthropic.com" +HELPER = ApiKeyHelper("lite auth print-token") +ORIGINAL = { + "theme": "dark", + "permissions": {"allow": ["Bash"]}, + "env": {"KEEP_ME": "1", "ANTHROPIC_API_KEY": "sk-ant-mine", "ANTHROPIC_BASE_URL": ANTHROPIC}, + "apiKeyHelper": "/usr/local/bin/lite auth print-token", + "model": "claude-opus-5", +} + + +def _set(path, value): + """A user edit: set (or with `_ABSENT`, remove) the key at a dotted path in the settings file.""" + + def edit(settings): + section, _, key = path.rpartition(".") + container = settings.setdefault(section, {}) if section else settings + if value is _ABSENT: + container.pop(key, None) + else: + container[key] = value + return settings + + return edit + + +_ABSENT = object() + + +class _Rig: + """One settings file plus receipt under tmp_path, driven through the public functions only.""" + + def __init__(self, tmp_path, initial): + self.settings = tmp_path / "claude" / "settings.json" + self.state = tmp_path / "state" / "claude_configure_state.json" + if initial is not None: + self.settings.parent.mkdir(parents=True) + self.settings.write_text(json.dumps(initial)) + + def read(self): + return json.loads(self.settings.read_text()) if self.settings.exists() else None + + def configure(self, credential=StaticToken("sk-virtual-key"), model=StartOn("claude-auto"), **kwargs): + configure_claude_settings(PROXY, credential, model, self.settings, self.state, (), **kwargs) + + def edit(self, *edits): + settings = self.read() + for apply in edits: + settings = apply(settings) + self.settings.write_text(json.dumps(settings)) + + def unconfigure(self): + return unconfigure_claude_settings(self.settings, self.state, ()) + + +# Each row: initial file, steps (configure kwargs dicts or edit callables) between the first configure +# and unconfigure, the expected file afterwards, and the expected outcome fields. Sequences that used +# to be one test each; the receipt's rules are what make them all come out right. +UNDO_SCENARIOS = { + "plain round trip": (ORIGINAL, [], ORIGINAL, {"kept": ()}), + "no file before": (None, [], None, {"file_removed": True}), + "no env before": ({"theme": "dark"}, [], {"theme": "dark"}, {}), + "null env before": ({"theme": "dark", "env": None}, [], {"theme": "dark", "env": None}, {}), + "empty env before": ({"theme": "dark", "env": {}}, [], {"theme": "dark", "env": {}}, {}), + "user edits stay and are named": ( + ORIGINAL, + [_set("env.ENABLE_TOOL_SEARCH", "false"), _set("model", "claude-sonnet-4-6")], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "claude-sonnet-4-6"}, + {"kept": {"env.ENABLE_TOOL_SEARCH", "model"}, "withheld": ()}, + ), + "user filled an env configure created": (None, [_set("env.MY_VAR", "mine")], {"env": {"MY_VAR": "mine"}}, {}), + "user deleted the file": (None, [lambda s: None], None, {"file_removed": True, "restored": (), "kept": ()}), + "user removed our key: neither restored nor kept": ( + ORIGINAL, + [_set("env.ANTHROPIC_AUTH_TOKEN", _ABSENT)], + ORIGINAL, + {"not_restored": {"env.ANTHROPIC_AUTH_TOKEN"}, "kept": ()}, + ), + "restored names only what changed": ( + {"model": "claude-opus-5"}, + [], + {"model": "claude-opus-5"}, + { + "restored": { + "env.ANTHROPIC_BASE_URL", + "env.ENABLE_TOOL_SEARCH", + "env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY", + "apiKeyHelper", + }, + "kept": (), + }, + {"credential": HELPER, "model": KeepModel()}, + ), + "repeat across credential kinds keeps the first snapshot": ( + ORIGINAL, + [ + {"credential": HELPER, "model": UnpinModel()}, + {"credential": StaticToken("sk-rotated"), "model": StartOn("claude-sonnet-4-6")}, + ], + ORIGINAL, + {}, + ), + "repeat without a model lets go of our pin, user had none": ({}, [{"model": UnpinModel()}], {}, {}), + "repeat without a model lets go of our pin, user had one": ( + {"model": "claude-opus-5"}, + [{"model": UnpinModel()}], + {"model": "claude-opus-5"}, + {}, + ), + "re-login keeps our pin": (None, [{"credential": HELPER, "model": KeepModel()}], None, {"file_removed": True}), + "edit between configures survives an unpin repeat": ( + ORIGINAL, + [ + _set("model", "my-favourite"), + _set("env.ENABLE_TOOL_SEARCH", "false"), + {"credential": HELPER, "model": UnpinModel()}, + ], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "my-favourite"}, + {"kept": {"env.ENABLE_TOOL_SEARCH", "model"}}, + ), + "edit between configures survives a re-login": ( + ORIGINAL, + [ + _set("model", "my-favourite"), + _set("env.ENABLE_TOOL_SEARCH", "false"), + {"credential": HELPER, "model": KeepModel()}, + ], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "my-favourite"}, + {"kept": {"env.ENABLE_TOOL_SEARCH", "model"}}, + ), + "edit between configures: a same-model repeat displaces it, so it is what comes back": ( + ORIGINAL, + [_set("model", "my-favourite"), _set("env.ENABLE_TOOL_SEARCH", "false"), {"credential": HELPER}], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "my-favourite"}, + {"kept": {"env.ENABLE_TOOL_SEARCH"}, "restored_includes": {"model"}}, + ), + "base URL changed since: credentials withheld, receipt kept": ( + ORIGINAL, + [_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000")], + {**ORIGINAL, "env": {"KEEP_ME": "1", "ANTHROPIC_BASE_URL": "http://other-proxy:4000"}, "apiKeyHelper": _ABSENT}, + { + "withheld": {("env.ANTHROPIC_API_KEY", ANTHROPIC), ("apiKeyHelper", ANTHROPIC)}, + "kept": {"env.ANTHROPIC_BASE_URL"}, + "receipt_kept": True, + }, + ), + "base URL changed and back: judged against the URL the restored file holds": ( + ORIGINAL, + [_set("env.ANTHROPIC_BASE_URL", ANTHROPIC)], + ORIGINAL, + {"withheld": ()}, + ), + "credential captured beside no URL goes back only beside no URL": ( + {"env": {"ANTHROPIC_API_KEY": "sk-default-endpoint"}}, + [_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000")], + {"env": {"ANTHROPIC_BASE_URL": "http://other-proxy:4000"}}, + { + "withheld": {("env.ANTHROPIC_API_KEY", "no ANTHROPIC_BASE_URL (Anthropic's default endpoint)")}, + "receipt_kept": True, + }, + ), + "restored document empty while a credential is withheld: file goes, receipt stays": ( + None, + [ + _set("env.ANTHROPIC_API_KEY", "sk-user"), + {"credential": HELPER, "model": KeepModel()}, + _set("env.ANTHROPIC_BASE_URL", _ABSENT), + ], + None, + {"withheld": {("env.ANTHROPIC_API_KEY", PROXY)}, "file_removed": True, "receipt_kept": True}, + {"credential": HELPER, "model": KeepModel()}, + ), + "a credential the user changed is kept, never also withheld": ( + ORIGINAL, + [_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000"), _set("apiKeyHelper", "/opt/mine/helper")], + { + **ORIGINAL, + "env": {"KEEP_ME": "1", "ANTHROPIC_BASE_URL": "http://other-proxy:4000"}, + "apiKeyHelper": "/opt/mine/helper", + }, + { + "withheld": {("env.ANTHROPIC_API_KEY", ANTHROPIC)}, + "kept": {"env.ANTHROPIC_BASE_URL", "apiKeyHelper"}, + "receipt_kept": True, + }, + ), +} + + +def _expected_file(expected): + if expected is None: + return None + return {k: v for k, v in expected.items() if v is not _ABSENT} + + +class TestConfigureAndUnconfigure: + """`configure_claude_settings` records how to undo itself; `unconfigure_claude_settings` undoes only that.""" + + @pytest.mark.parametrize("scenario", UNDO_SCENARIOS.values(), ids=UNDO_SCENARIOS.keys()) + def test_undo_matrix(self, tmp_path, scenario): + initial, steps, expected, outcome_expectations, *first = scenario + rig = _Rig(tmp_path, initial) + rig.configure(**(first[0] if first else {})) + for step in steps: + if isinstance(step, dict): + rig.configure(**step) + elif rig.settings.exists() and step(json.loads(rig.settings.read_text())) is None: + rig.settings.unlink() + else: + rig.edit(step) + + outcome = rig.unconfigure() + + assert rig.read() == _expected_file(expected) + assert rig.state.exists() == outcome_expectations.get("receipt_kept", False) + for field, want in outcome_expectations.items(): + if field == "withheld": + assert {(item.key, item.endpoint) for item in outcome.withheld} == set(want) + elif field == "not_restored": + assert not set(want) & set(outcome.restored) and not set(want) & set(outcome.kept) + elif field == "restored_includes": + assert set(want) <= set(outcome.restored) + elif field in ("restored", "kept"): + assert set(getattr(outcome, field)) == set(want) + elif field != "receipt_kept": + assert getattr(outcome, field) == want + assert not {item.key for item in outcome.withheld} & set(outcome.kept) + + def test_configure_writes_owner_only_and_the_receipt_never_holds_the_key(self, tmp_path): + rig = _Rig(tmp_path, ORIGINAL) + rig.configure(credential=StaticToken("sk-virtual-key-never-on-disk-twice")) + configured = rig.read() + assert configured["env"]["ANTHROPIC_AUTH_TOKEN"] == "sk-virtual-key-never-on-disk-twice" + assert configured["env"]["ANTHROPIC_BASE_URL"] == PROXY and configured["model"] == "claude-auto" + assert "ANTHROPIC_API_KEY" not in configured["env"] and "apiKeyHelper" not in configured + assert stat.S_IMODE(rig.settings.stat().st_mode) == 0o600 == stat.S_IMODE(rig.state.stat().st_mode) + assert "sk-virtual-key-never-on-disk-twice" not in rig.state.read_text() + + def test_withheld_credentials_come_back_once_the_url_points_at_their_server_again(self, tmp_path): + # The kept receipt owns only the withheld slots: the second unconfigure restores exactly those. + rig = _Rig(tmp_path, ORIGINAL) + rig.configure() + rig.edit(_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000")) + rig.unconfigure() + rig.edit(_set("env.ANTHROPIC_BASE_URL", ANTHROPIC), _set("theme", "light")) + outcome = rig.unconfigure() + assert rig.read() == {**ORIGINAL, "theme": "light"} + assert set(outcome.restored) == {"env.ANTHROPIC_API_KEY", "apiKeyHelper"} + assert outcome.kept == () and outcome.withheld == () and not rig.state.exists() + + @pytest.mark.parametrize( + ("path", "value", "repeat_credential"), + [ + ("env.ANTHROPIC_API_KEY", "sk-user-added-later", HELPER), + ("env.ANTHROPIC_AUTH_TOKEN", "sk-users-own-token", HELPER), + ("apiKeyHelper", "/opt/mine/helper", StaticToken("sk-rotated")), + ], + ids=["user-adds-api-key", "user-replaces-our-token", "user-sets-own-helper"], + ) + def test_a_credential_the_user_set_between_two_configures_is_what_comes_back( + self, tmp_path, path, value, repeat_credential + ): + # The repeat's merge clears the slot, so the displaced value is snapshotted and is what returns; + # it was set while the file pointed at the proxy, so it returns once the file points there again. + rig = _Rig(tmp_path, {"theme": "dark"}) + rig.configure(credential=HELPER, model=KeepModel()) + rig.edit(_set(path, value)) + rig.configure(credential=repeat_credential, model=KeepModel()) + assert not _lookup(rig.read(), path) + + outcome = rig.unconfigure() + assert [(item.key, item.endpoint) for item in outcome.withheld] == [(path, PROXY)] + assert rig.read() == {"theme": "dark"} and rig.state.exists() + rig.settings.write_text(json.dumps({"theme": "dark", "env": {"ANTHROPIC_BASE_URL": PROXY}})) + outcome = rig.unconfigure() + assert _lookup(rig.read(), path) == value + assert outcome.restored == (path,) and outcome.withheld == () and not rig.state.exists() + + def test_a_receipt_commit_that_fails_leaves_no_staged_token_behind(self, tmp_path): + rig = _Rig(tmp_path, {}) + + def commit_receipt_fails(staged, path): + if path == str(rig.state): + os.unlink(staged) + raise OSError("receipt rename failed") + commit_staged_json(staged, path) + + with pytest.raises(ClaudeSettingsError, match=r"Could not write .*receipt rename failed"): + rig.configure(credential=StaticToken("sk-never-left-in-a-temp-file"), commit=commit_receipt_fails) + assert not list(rig.settings.parent.glob(".tmp-*")) and not list(rig.state.parent.glob(".tmp-*")) + assert rig.read() == {} and not rig.state.exists() + + @pytest.mark.parametrize("configured_before", [False, True], ids=["first-configure", "repeat-configure"]) + def test_a_settings_commit_that_fails_after_the_receipt_landed_puts_the_receipt_back( + self, tmp_path, configured_before + ): + # The two renames are not atomic: a settings rename that fails after the receipt landed must + # not leave a receipt describing settings that were never written. + rig = _Rig(tmp_path, ORIGINAL) + if configured_before: + rig.configure() + receipt_before = rig.state.read_text() if configured_before else None + settings_before = rig.settings.read_text() + + def commit_settings_fails(staged, path): + if path == str(rig.settings): + os.unlink(staged) + raise OSError("rename failed") + commit_staged_json(staged, path) + + with pytest.raises(ClaudeSettingsError, match="rename failed"): + rig.configure(credential=StaticToken("sk-rotated"), commit=commit_settings_fails) + assert rig.settings.read_text() == settings_before + assert (rig.state.read_text() if rig.state.exists() else None) == receipt_before + if configured_before: + rig.unconfigure() + assert rig.read() == ORIGINAL + + def test_a_failed_repeat_configure_leaves_the_earlier_undo_intact(self, tmp_path): + rig = _Rig(tmp_path, ORIGINAL) + rig.configure() + receipt_before = rig.state.read_text() + rig.settings.parent.chmod(0o500) + try: + with pytest.raises(ClaudeSettingsError, match="Could not write"): + rig.configure(credential=StaticToken("sk-rotated")) + finally: + rig.settings.parent.chmod(0o700) + assert rig.state.read_text() == receipt_before and not list(rig.state.parent.glob(".tmp-*")) + assert rig.read()["env"]["ANTHROPIC_AUTH_TOKEN"] == "sk-virtual-key" + rig.unconfigure() + assert rig.read() == ORIGINAL + + def test_unconfigure_reports_a_receipt_it_cannot_remove_as_a_settings_error(self, tmp_path): + rig = _Rig(tmp_path, ORIGINAL) + rig.configure() + rig.state.parent.chmod(0o500) + try: + with pytest.raises(ClaudeSettingsError, match="Could not remove"): + rig.unconfigure() + finally: + rig.state.parent.chmod(0o700) + + def test_configure_writes_through_a_symlinked_settings_file(self, tmp_path): + target = tmp_path / "dotfiles" / "settings.json" + target.parent.mkdir() + target.write_text(json.dumps({"theme": "dark"})) + link = tmp_path / "settings.json" + link.symlink_to(target) + configure_claude_settings(PROXY, StaticToken("sk-virtual-key"), UnpinModel(), link, tmp_path / "state.json", ()) + assert link.is_symlink() + assert json.loads(target.read_text())["env"]["ANTHROPIC_AUTH_TOKEN"] == "sk-virtual-key" + + @pytest.mark.parametrize("operation", ["configure", "unconfigure"]) + def test_refuses_while_a_temporary_owner_holds_a_backup(self, paths, tmp_path, operation): + settings_path, backup_path = paths + backup_path.write_text("{}") + owners = _owners(backup_path) + state = tmp_path / "state.json" + attempt = ( + (lambda: configure_claude_settings(PROXY, StaticToken("k"), UnpinModel(), settings_path, state, owners)) + if operation == "configure" + else (lambda: unconfigure_claude_settings(settings_path, state, owners)) + ) + with pytest.raises(ClaudeSettingsError, match="lite down"): + attempt() + assert not settings_path.exists() + + def test_unconfigure_without_a_receipt_is_an_error_not_a_silent_no_op(self, tmp_path): + with pytest.raises(ClaudeSettingsError, match="nothing to undo"): + _Rig(tmp_path, None).unconfigure() + + +def _lookup(settings, path): + section, _, key = path.rpartition(".") + return (settings.get(section) or {}).get(key) if section else settings.get(key) diff --git a/tests/test_litellm/proxy/client/cli/test_configure_commands.py b/tests/test_litellm/proxy/client/cli/test_configure_commands.py new file mode 100644 index 00000000000..1631ba7f49e --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/test_configure_commands.py @@ -0,0 +1,331 @@ +import json +import os +import stat + +import click +import pytest +import requests +import responses +from click.testing import CliRunner + +from litellm.proxy.client.cli import cli +from litellm.proxy.client.cli.commands import configure as configure_module +from litellm.proxy.client.cli.commands.claude_settings import SettingsFileOwner +from litellm.proxy.client.cli.commands.configure import configure_claude, configure_group, interactive_configure + +PROXY = "http://proxy.test:4000" +VALID_KEY = "sk-virtual-key" +LISTED_MODELS = ("claude-auto", "gpt-5.6-luna") + + +def _mock_models(): + responses.get( + f"{PROXY}/v1/models", + json={"data": [{"id": model, "object": "model"} for model in LISTED_MODELS]}, + match=[responses.matchers.header_matcher({"Authorization": f"Bearer {VALID_KEY}"})], + ) + responses.get(f"{PROXY}/v1/models", status=401) + + +@pytest.fixture +def paths(monkeypatch, tmp_path): + settings_path = tmp_path / "claude" / "settings.json" + state_path = tmp_path / "litellm" / "claude_configure_state.json" + monkeypatch.setattr(configure_module, "CLAUDE_SETTINGS_PATH", settings_path) + monkeypatch.setattr(configure_module, "CONFIGURE_STATE_PATH", state_path) + return settings_path, state_path + + +@pytest.fixture +def lite_on_path(monkeypatch, tmp_path): + """A real `lite` executable on PATH, so the apiKeyHelper command resolves without patching.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + lite = bin_dir / "lite" + lite.write_text("#!/bin/sh\nexit 0\n") + lite.chmod(lite.stat().st_mode | stat.S_IXUSR) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}") + return str(lite) + + +@pytest.fixture +def runner(): + return CliRunner() + + +@pytest.fixture +def lite_up_backup(monkeypatch, tmp_path): + """A `lite up` session holding its backup, the local precondition every settings write refuses on.""" + backup = tmp_path / "claude_settings_backup.json" + backup.write_text("{}") + monkeypatch.setattr(configure_module, "SETTINGS_FILE_OWNERS", (SettingsFileOwner(backup, "lite up", "lite down"),)) + return backup + + +def _configure(runner, *args): + return runner.invoke(cli, ["--base-url", PROXY, "configure", "claude", *args]) + + +class TestConfigureClaudeWithAVirtualKey: + @responses.activate + def test_writes_settings_and_reports_without_echoing_the_key(self, runner, paths): + _mock_models() + settings_path, state_path = paths + result = _configure(runner, "--api-key", VALID_KEY, "--model", "claude-auto") + assert result.exit_code == 0, result.output + written = json.loads(settings_path.read_text()) + assert written["env"]["ANTHROPIC_BASE_URL"] == PROXY + assert written["env"]["ANTHROPIC_AUTH_TOKEN"] == VALID_KEY + assert written["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1" + assert written["model"] == "claude-auto" + assert "ANTHROPIC_DEFAULT_SONNET_MODEL" not in written["env"] + assert state_path.exists() + assert VALID_KEY not in result.output + assert "Starting model: claude-auto" in result.output + assert "1 of the proxy's 2 models" in result.output + assert "lite unconfigure claude" in result.output + assert len(responses.calls) == 1 + + @responses.activate + def test_takes_the_key_from_the_global_option_and_keeps_claude_codes_default(self, runner, paths): + _mock_models() + settings_path, _ = paths + result = runner.invoke(cli, ["--base-url", PROXY, "--api-key", VALID_KEY, "configure", "claude"]) + assert result.exit_code == 0, result.output + written = json.loads(settings_path.read_text()) + assert written["env"]["ANTHROPIC_AUTH_TOKEN"] == VALID_KEY + assert "model" not in written + assert "Starting model: not pinned" in result.output + + @responses.activate + def test_refuses_a_model_the_proxy_does_not_list(self, runner, paths): + _mock_models() + settings_path, _ = paths + result = _configure(runner, "--api-key", VALID_KEY, "--model", "claude-nope") + assert result.exit_code != 0 + assert "'claude-nope' is not served" in result.output + assert "claude-auto, gpt-5.6-luna" in result.output + assert not settings_path.exists() + + @responses.activate + def test_refuses_a_key_the_proxy_rejects(self, runner, paths): + _mock_models() + settings_path, _ = paths + result = _configure(runner, "--api-key", "sk-wrong") + assert result.exit_code != 0 + assert "rejected your key (HTTP 401)" in result.output + assert not settings_path.exists() + + @responses.activate + @pytest.mark.parametrize( + ("mock", "expected", "unexpected"), + [ + ( + lambda: responses.get(f"{PROXY}/v1/models", body=requests.ConnectionError("refused")), + "Is the proxy at", + "answered", + ), + ( + lambda: responses.get(f"{PROXY}/v1/models", status=500), + "The proxy at http://proxy.test:4000 answered", + "Is the proxy at", + ), + ( + lambda: responses.get(f"{PROXY}/v1/models", body="not json"), + "answered, so check that it is a LiteLLM proxy", + "Is the proxy at", + ), + ( + lambda: responses.get(f"{PROXY}/v1/models", json={"data": []}), + "Claude Code would have nothing to run", + "Is the proxy at", + ), + ], + ids=["unreachable", "http-500", "non-json-body", "empty-list"], + ) + def test_the_listing_hint_matches_how_the_listing_failed(self, runner, paths, mock, expected, unexpected): + # Only a proxy that never answered gets the "is it running" question; a 500, a non-JSON body or an + # empty list prove it is up, and the hint says so instead. + mock() + settings_path, _ = paths + result = _configure(runner, "--api-key", VALID_KEY) + assert result.exit_code != 0 + assert expected in result.output and unexpected not in result.output + assert not settings_path.exists() + + @responses.activate + @pytest.mark.parametrize("entry", ["virtual-key", "login", "interactive"]) + def test_refuses_while_lite_up_holds_a_backup_before_any_login_or_request( + self, runner, paths, monkeypatch, lite_up_backup, entry + ): + _mock_models() + + def login_must_not_run(ctx): + raise AssertionError("the local precondition must be checked before a login is attempted") + + monkeypatch.setattr(configure_module, "ensure_fresh_login", login_must_not_run) + if entry == "interactive": + ctx = click.Context(configure_group, obj={"base_url": PROXY, "api_key": None}) + with pytest.raises(click.ClickException, match="lite down"): + interactive_configure(ctx, pick_targets=lambda: ("claude",), pick_model=lambda listed: None) + else: + args = ["--api-key", VALID_KEY] if entry == "virtual-key" else [] + result = runner.invoke(configure_claude, args, obj={"base_url": PROXY, "api_key": None}) + assert result.exit_code != 0 and "lite down" in result.output + assert len(responses.calls) == 0 + assert not paths[0].exists() + + @responses.activate + def test_says_so_when_the_key_is_written_through_a_symlink(self, runner, paths, tmp_path): + _mock_models() + settings_path, _ = paths + target = tmp_path / "dotfiles" / "settings.json" + target.parent.mkdir() + target.write_text("{}") + settings_path.parent.mkdir(parents=True) + settings_path.symlink_to(target) + result = _configure(runner, "--api-key", VALID_KEY) + assert result.exit_code == 0, result.output + assert "keep it out of version control" in result.output + assert json.loads(target.read_text())["env"]["ANTHROPIC_AUTH_TOKEN"] == VALID_KEY + + +class TestConfigureClaudeWithTheLogin: + def _stored_login(self, monkeypatch): + monkeypatch.setattr(configure_module, "ensure_fresh_login", lambda ctx: None) + monkeypatch.setattr(configure_module, "get_stored_api_key", lambda expected_base_url, vault: VALID_KEY) + + @responses.activate + def test_uses_the_login_through_the_helper_and_writes_no_secret(self, runner, paths, monkeypatch, lite_on_path): + _mock_models() + self._stored_login(monkeypatch) + settings_path, _ = paths + result = runner.invoke( + configure_claude, + ["--model", "claude-auto"], + obj={"base_url": PROXY, "api_key": VALID_KEY, "api_key_from_token_file": True}, + ) + assert result.exit_code == 0, result.output + written = json.loads(settings_path.read_text()) + assert written["apiKeyHelper"] == f"{lite_on_path} --base-url {PROXY} auth print-token" + assert "ANTHROPIC_AUTH_TOKEN" not in written["env"] + assert written["model"] == "claude-auto" + assert VALID_KEY not in settings_path.read_text() + assert "read through apiKeyHelper" in result.output + + @responses.activate + def test_an_explicit_key_still_wins_over_a_stored_login(self, runner, paths, monkeypatch, lite_on_path): + _mock_models() + self._stored_login(monkeypatch) + settings_path, _ = paths + result = runner.invoke( + configure_claude, + ["--api-key", VALID_KEY], + obj={"base_url": PROXY, "api_key": "sk-login-jwt", "api_key_from_token_file": True}, + ) + assert result.exit_code == 0, result.output + written = json.loads(settings_path.read_text()) + assert written["env"]["ANTHROPIC_AUTH_TOKEN"] == VALID_KEY and "apiKeyHelper" not in written + + +class TestInteractiveConfigure: + @responses.activate + def test_asks_for_targets_and_a_starting_model_then_configures(self, paths): + _mock_models() + settings_path, _ = paths + asked = {} + + def pick_model(listed): + asked["listed"] = tuple(listed) + return "claude-auto" + + ctx = click.Context( + configure_group, obj={"base_url": PROXY, "api_key": VALID_KEY, "api_key_from_token_file": False} + ) + interactive_configure(ctx, pick_targets=lambda: ("claude",), pick_model=pick_model) + assert asked["listed"] == LISTED_MODELS + assert json.loads(settings_path.read_text())["model"] == "claude-auto" + + def test_does_nothing_when_claude_code_is_not_picked(self, paths): + settings_path, _ = paths + ctx = click.Context( + configure_group, obj={"base_url": PROXY, "api_key": VALID_KEY, "api_key_from_token_file": False} + ) + interactive_configure(ctx, pick_targets=lambda: (), pick_model=lambda listed: None) + assert not settings_path.exists() + + def test_bare_configure_without_a_terminal_names_the_non_interactive_command(self, runner, paths): + result = runner.invoke(cli, ["--base-url", PROXY, "configure"]) + assert result.exit_code != 0 + assert "lite configure claude --api-key" in result.output + + +class TestUnconfigureClaude: + @responses.activate + def test_restores_the_original_file_and_removes_the_receipt(self, runner, paths): + _mock_models() + settings_path, state_path = paths + settings_path.parent.mkdir(parents=True) + original = {"theme": "dark", "model": "claude-opus-5"} + settings_path.write_text(json.dumps(original)) + assert _configure(runner, "--api-key", VALID_KEY, "--model", "claude-auto").exit_code == 0 + + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code == 0, result.output + assert json.loads(settings_path.read_text()) == original + assert not state_path.exists() + assert "Restored in" in result.output and "model" in result.output + assert "ANTHROPIC_API_KEY" not in result.output, "a key that never existed was not restored" + + @responses.activate + def test_a_file_only_configure_created_is_reported_removed_not_restored(self, runner, paths): + _mock_models() + settings_path, _ = paths + assert _configure(runner, "--api-key", VALID_KEY).exit_code == 0 + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code == 0, result.output + assert not settings_path.exists() + assert "No settings file remains" in result.output and "Restored" not in result.output + + @responses.activate + def test_says_when_nothing_was_still_ours_and_names_what_it_kept(self, runner, paths): + _mock_models() + settings_path, _ = paths + settings_path.parent.mkdir(parents=True) + settings_path.write_text(json.dumps({"theme": "dark"})) + assert _configure(runner, "--api-key", VALID_KEY, "--model", "claude-auto").exit_code == 0 + edited = json.loads(settings_path.read_text()) + edited["env"] = {key: f"{value}-edited" for key, value in edited["env"].items()} + edited["model"] = "mine" + settings_path.write_text(json.dumps(edited)) + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code == 0, result.output + assert "Nothing in" in result.output and "was still ours to restore" in result.output + assert "Left as you changed them since:" in result.output and "model" in result.output + + @responses.activate + def test_names_the_server_a_withheld_credential_was_captured_with_and_keeps_the_receipt(self, runner, paths): + _mock_models() + settings_path, state_path = paths + settings_path.parent.mkdir(parents=True) + settings_path.write_text( + json.dumps({"env": {"ANTHROPIC_BASE_URL": "https://api.anthropic.com", "ANTHROPIC_API_KEY": "sk-ant"}}) + ) + assert _configure(runner, "--api-key", VALID_KEY).exit_code == 0 + edited = json.loads(settings_path.read_text()) + edited["env"]["ANTHROPIC_BASE_URL"] = "http://other-proxy:4000" + settings_path.write_text(json.dumps(edited)) + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code == 0, result.output + assert "env.ANTHROPIC_API_KEY (captured with https://api.anthropic.com)" in result.output + assert str(state_path) in result.output and state_path.exists() + assert "sk-ant" not in result.output + + def test_refuses_while_lite_up_holds_a_backup(self, runner, paths, lite_up_backup): + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code != 0 and "lite down" in result.output + + def test_without_a_receipt_it_fails_loudly(self, runner, paths): + result = runner.invoke(cli, ["unconfigure", "claude"]) + assert result.exit_code != 0 + assert "nothing to undo" in result.output diff --git a/tests/test_litellm/proxy/client/cli/test_pi.py b/tests/test_litellm/proxy/client/cli/test_pi.py index 68c0ac70064..1c2da514ee2 100644 --- a/tests/test_litellm/proxy/client/cli/test_pi.py +++ b/tests/test_litellm/proxy/client/cli/test_pi.py @@ -4,9 +4,11 @@ import stat from concurrent.futures import ThreadPoolExecutor from pathlib import Path +import pytest import requests from litellm.proxy.client.cli.commands.pi import ( + ListingFailure, ModelLimits, PiSyncError, fetch_model_ids, @@ -28,6 +30,10 @@ class _FakeResponse: return self._payload +def _refused(*args, **kwargs): + raise requests.ConnectionError("refused") + + class TestFetchModelIds: def test_returns_ids_in_proxy_order_deduped(self): captured = {} @@ -53,9 +59,7 @@ class TestFetchModelIds: assert "Could not list models" in result.message def test_non_200_is_a_value(self): - result = fetch_model_ids( - "http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(500) - ) + result = fetch_model_ids("http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(500)) assert isinstance(result, PiSyncError) assert "HTTP 500" in result.message @@ -75,6 +79,22 @@ class TestFetchModelIds: ) assert isinstance(result, PiSyncError) assert "no models" in result.message + assert result.kind is ListingFailure.EMPTY + + @pytest.mark.parametrize( + ("get", "kind"), + [ + (_refused, ListingFailure.UNREACHABLE), + (lambda *a, **k: _FakeResponse(401), ListingFailure.REJECTED), + (lambda *a, **k: _FakeResponse(403), ListingFailure.REJECTED), + (lambda *a, **k: _FakeResponse(500), ListingFailure.OTHER), + (lambda *a, **k: _FakeResponse(200), ListingFailure.BAD_BODY), + ], + ids=["unreachable", "401", "403", "500", "bad-body"], + ) + def test_the_failure_kind_is_decided_where_the_response_is_classified(self, get, kind): + result = fetch_model_ids("http://localhost:4000", "sk-key", get=get) + assert isinstance(result, PiSyncError) and result.kind is kind class TestFetchModelLimits: diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index aead1764b0e..b607b1a3db7 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -11,11 +11,11 @@ from click.testing import CliRunner from litellm.proxy.client.cli.commands import up as up_module from litellm.proxy.client.cli.commands.agents import AgentRunError -from litellm.proxy.client.cli.commands.claude_settings import ClaudeSettingsError +from litellm.proxy.client.cli.commands.claude_settings import ApiKeyHelper, ClaudeSettingsError from litellm.proxy.client.cli.commands.up import ( BackupRecord, UpError, - _ensure_fresh_login, + ensure_fresh_login, down, load_json_or_empty, merge_claude_settings, @@ -40,12 +40,12 @@ def _patch_paths(monkeypatch, tmp_path): class TestMergeClaudeSettings: def test_preserves_unrelated_top_level_keys(self): - merged = merge_claude_settings({"theme": "dark"}, "http://localhost:4000", "helper") + merged = merge_claude_settings({"theme": "dark"}, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["theme"] == "dark" def test_preserves_unrelated_env_keys(self): settings = {"env": {"SOME_OTHER_VAR": "value"}} - merged = merge_claude_settings(settings, "http://localhost:4000", "helper") + merged = merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["env"]["SOME_OTHER_VAR"] == "value" def test_overrides_base_url_and_helper(self): @@ -53,7 +53,7 @@ class TestMergeClaudeSettings: "env": {"ANTHROPIC_BASE_URL": "https://old.example.com"}, "apiKeyHelper": "old-helper", } - merged = merge_claude_settings(settings, "http://localhost:4000/", "new-helper") + merged = merge_claude_settings(settings, "http://localhost:4000/", ApiKeyHelper("new-helper")) assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://localhost:4000" assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true" assert merged["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1" @@ -61,21 +61,21 @@ class TestMergeClaudeSettings: def test_preserves_existing_gateway_model_discovery(self): settings = {"env": {"CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY": "0"}} - merged = merge_claude_settings(settings, "http://localhost:4000", "helper") + merged = merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["env"]["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "0" def test_preserves_existing_tool_search(self): settings = {"env": {"ENABLE_TOOL_SEARCH": "false"}} - merged = merge_claude_settings(settings, "http://localhost:4000", "helper") + merged = merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false" def test_drops_stray_api_key(self): settings = {"env": {"ANTHROPIC_API_KEY": "leaked-key"}} - merged = merge_claude_settings(settings, "http://localhost:4000", "helper") + merged = merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert "ANTHROPIC_API_KEY" not in merged["env"] def test_works_from_empty_settings(self): - merged = merge_claude_settings({}, "http://localhost:4000", "helper") + merged = merge_claude_settings({}, "http://localhost:4000", ApiKeyHelper("helper")) assert merged["env"] == { "ANTHROPIC_BASE_URL": "http://localhost:4000", "ENABLE_TOOL_SEARCH": "true", @@ -85,7 +85,7 @@ class TestMergeClaudeSettings: def test_does_not_mutate_input(self): settings = {"env": {"FOO": "bar"}} - merge_claude_settings(settings, "http://localhost:4000", "helper") + merge_claude_settings(settings, "http://localhost:4000", ApiKeyHelper("helper")) assert settings == {"env": {"FOO": "bar"}} @@ -327,7 +327,7 @@ class TestEnsureFreshLogin: monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) login_calls = _capture_login(monkeypatch) - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) assert login_calls == [] @@ -339,7 +339,7 @@ class TestEnsureFreshLogin: monkeypatch, on_login=lambda: store.log_in({"key": "sk-b", "base_url": "http://proxy-b:4000"}, "sk-b") ) - _ensure_fresh_login(_make_ctx("http://proxy-b:4000")) + ensure_fresh_login(_make_ctx("http://proxy-b:4000")) assert login_calls == [("http://proxy-b:4000", False)] assert store.key_requests == ["http://proxy-b:4000", "http://proxy-b:4000"] @@ -353,7 +353,7 @@ class TestEnsureFreshLogin: on_login=lambda: store.log_in({"key": "sk-a", "base_url": "http://proxy-a:4000"}, "sk-a"), ) - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) assert login_calls == [("http://proxy-a:4000", False)] @@ -363,7 +363,7 @@ class TestEnsureFreshLogin: monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) with pytest.raises(UpError, match="Run `lite login` first"): - _ensure_fresh_login(_make_ctx("http://proxy-b:4000")) + ensure_fresh_login(_make_ctx("http://proxy-b:4000")) def test_trusts_a_pkce_credential_that_was_renewed_on_the_way_in(self, monkeypatch): """A --pkce key inside its freshness buffer is renewed by `get_stored_api_key`, so `lite up` @@ -377,7 +377,7 @@ class TestEnsureFreshLogin: ) login_calls = _capture_login(monkeypatch) - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) assert login_calls == [] assert store.key_requests == ["http://proxy-a:4000"] @@ -390,7 +390,7 @@ class TestEnsureFreshLogin: on_login=lambda: store.log_in(_pkce_record("http://proxy-a:4000", seconds_left=86_400), "sk-pkce-fresh"), ) - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) assert login_calls == [("http://proxy-a:4000", True)] @@ -399,7 +399,7 @@ class TestEnsureFreshLogin: _FakeTokenStore(monkeypatch, _pkce_record("http://proxy-a:4000", seconds_left=-10), {}) with pytest.raises(UpError, match="Run `lite login --pkce` first"): - _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + ensure_fresh_login(_make_ctx("http://proxy-a:4000")) def test_trusts_the_key_the_cli_group_already_resolved_instead_of_reading_the_token_file_again( self, monkeypatch @@ -409,7 +409,7 @@ class TestEnsureFreshLogin: store = _FakeTokenStore(monkeypatch, _pkce_record("http://proxy-a:4000", seconds_left=86_400), {}) login_calls = _capture_login(monkeypatch) - _ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key="sk-pkce-renewed-by-the-group")) + ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key="sk-pkce-renewed-by-the-group")) assert login_calls == [] assert store.key_requests == [] @@ -423,7 +423,7 @@ class TestEnsureFreshLogin: ) with pytest.raises(UpError, match="Run `lite login --pkce` first"): - _ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key=None)) + ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key=None)) assert store.key_requests == [] @@ -435,7 +435,7 @@ class TestEnsureFreshLogin: on_login=lambda: store.log_in(_pkce_record("http://proxy-a:4000", seconds_left=86_400), "sk-pkce-fresh"), ) - _ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key=None)) + ensure_fresh_login(_make_group_ctx("http://proxy-a:4000", api_key=None)) assert login_calls == [("http://proxy-a:4000", True)] assert store.key_requests == ["http://proxy-a:4000"] From 3523f3731cb356bcdf642182ea6b971d65291f1b Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:01:58 -0700 Subject: [PATCH 266/411] fix(mcp): bind inherited preview credentials to the saved origin --- .../mcp_server/rest_endpoints.py | 13 ++++++- .../mcp_server/test_rest_endpoints.py | 39 ++++++++++++++++++- .../_components/editToolPreview.test.ts | 22 ++++++++++- .../_components/editToolPreview.ts | 16 +++++++- .../_components/mcp_server_edit.tsx | 11 ++++-- 5 files changed, 91 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index a72465f16f4..a116c09d844 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1255,7 +1255,18 @@ if MCP_AVAILABLE: MCPRequestHandler, ) - request: Final = _inherit_credentials_from_existing_server(new_mcp_server_request) + saved_server: Final = ( + global_mcp_server_manager.get_mcp_server_by_id(new_mcp_server_request.server_id) + if new_mcp_server_request.server_id + else None + ) + saved_origin: Final = _redact_mcp_resource_url(saved_server.url) if saved_server else None + may_inherit: Final = new_mcp_server_request.auth_type not in _STAGED_AUTH_VALUE_AUTH_TYPES or ( + saved_origin is not None and saved_origin == _redact_mcp_resource_url(new_mcp_server_request.url) + ) + request: Final = ( + _inherit_credentials_from_existing_server(new_mcp_server_request) if may_inherit else new_mcp_server_request + ) mcp_auth_header: Final = ( request.credentials.get("auth_value") if request.auth_type in _STAGED_AUTH_VALUE_AUTH_TYPES and isinstance(request.credentials, dict) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 771a73c8ad0..ee5dabf043f 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -128,7 +128,7 @@ class TestExecuteWithMcpClient: payload: Final = NewMCPServerRequest( server_id=saved.server_id, server_name="edited", - url="https://edited.example/mcp", + url="https://stored.example/corrected-mcp", transport=MCPTransport.sse, auth_type=auth_type, credentials={"auth_value": auth_value} if auth_value is not None else None, @@ -146,13 +146,48 @@ class TestExecuteWithMcpClient: oauth2_headers=staged.oauth2_headers, ) assert result == { - "url": "https://edited.example/mcp", + "url": "https://stored.example/corrected-mcp", "transport": MCPTransport.sse, "headers": {"X-Preview": "edited", **expected_auth}, } assert manager.get_mcp_server_by_id(saved.server_id) is saved assert saved.url == "https://stored.example/mcp" + @pytest.mark.parametrize( + "url", ("https://other.example/mcp", "http://stored.example/mcp", "https://stored.example:8443/mcp") + ) + @pytest.mark.parametrize("explicit_credential", (None, "preview:explicit")) + def test_static_preview_does_not_inherit_credentials_across_origins( + self, monkeypatch: pytest.MonkeyPatch, url: str, explicit_credential: str | None + ) -> None: + from starlette.datastructures import Headers + + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.management_endpoints import mcp_management_endpoints + + saved: Final = MCPServer( + server_id="saved-preview-server", + name="saved", + url="https://stored.example/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.basic, + authentication_token="preview:stored", + ) + manager: Final = MCPServerManager() + manager.registry = {saved.server_id: saved} + monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", manager) + monkeypatch.setattr(mcp_management_endpoints, "global_mcp_server_manager", manager) + payload: Final = NewMCPServerRequest( + server_id=saved.server_id, + url=url, + transport=MCPTransport.http, + auth_type=MCPAuth.basic, + credentials={"auth_value": explicit_credential} if explicit_credential else None, + ) + staged: Final = rest_endpoints._stage_server_test(payload, Headers()) + assert staged.mcp_auth_header == explicit_credential + assert staged.request.credentials == payload.credentials + @pytest.mark.asyncio async def test_redacts_stack_trace(self, monkeypatch): async def fake_create_client(*args, **kwargs): diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.test.ts index 9dcfc1a470e..c7147466d43 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.test.ts @@ -14,10 +14,10 @@ describe("getEditToolPreview", () => { }); it("previews URL changes with the existing server credential left for server-side inheritance", () => { - expect(getEditToolPreview({ ...saved, url: "https://correct.example/mcp" }, saved)).toEqual({ + expect(getEditToolPreview({ ...saved, url: "https://example.com/corrected-mcp" }, saved)).toEqual({ kind: "preview", config: { - url: "https://correct.example/mcp", + url: "https://example.com/corrected-mcp", transport: "http", auth_type: "basic", static_headers: { "X-Tenant": "original" }, @@ -26,6 +26,24 @@ describe("getEditToolPreview", () => { }); }); + it.each(["https://other.example/mcp", "http://example.com/mcp", "https://example.com:8443/mcp"])( + "requires explicit credentials for a changed origin: %s", + (url) => { + expect(getEditToolPreview({ ...saved, url, static_headers: [] }, saved)).toEqual({ + kind: "incomplete", + message: expect.stringContaining("origin changed"), + }); + const explicit = { ...saved, url, static_headers: [], credentials: { auth_value: "new:secret" } }; + expect(getEditToolPreview(explicit, saved).kind).toBe("preview"); + }, + ); + + it("does not automatically send saved static headers to a new origin", () => { + expect(getEditToolPreview({ ...saved, url: "https://other.example/mcp", auth_type: "none" }, saved).kind).toBe( + "incomplete", + ); + }); + it("uses edited static headers and only the static auth value", () => { expect( getEditToolPreview( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.ts index eb46c33c4e7..6dea7d7d18e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editToolPreview.ts @@ -21,7 +21,7 @@ const connectionConfig = (values: Readonly>) => { type EditToolPreview = | { readonly kind: "saved" } - | { readonly kind: "incomplete" } + | { readonly kind: "incomplete"; readonly message?: string } | { readonly kind: "preview"; readonly config: ReturnType }; export const getEditToolPreview = ( @@ -49,5 +49,19 @@ export const getEditToolPreview = ( if (!validUrl || missingNewCredential || incompleteHeaders) { return { kind: "incomplete" }; } + const savedConfig = connectionConfig(initialValues); + const changedOrigin = + !URL.canParse(savedConfig.url) || new URL(config.url).origin !== new URL(savedConfig.url).origin; + const reusesHeader = Object.entries(config.static_headers).some( + ([key, value]) => savedConfig.static_headers[key] === value, + ); + const needsSavedCredential = AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(config.auth_type) && !config.credentials; + if (changedOrigin && (needsSavedCredential || reusesHeader)) { + return { + kind: "incomplete", + message: + "The server origin changed. Enter credentials and replace or remove saved static headers to preview tools.", + }; + } return { kind: "preview", config }; }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx index 5316a22ce27..2a37029a2c4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx @@ -454,19 +454,22 @@ const MCPServerEdit: React.FC = ({ const toolPreviewKey = JSON.stringify(toolPreview); useEffect(() => { - let active = true; + const controller = new AbortController(); setTools([]); setToolsError(null); setIsLoadingTools(false); if (!accessToken || !mcpServer.server_id) return; if (toolPreview.kind === "incomplete") { - setToolsError("Complete the URL, authentication, and header settings to load tools."); + setToolsError(toolPreview.message ?? "Complete the URL, authentication, and header settings to load tools."); return; } setIsLoadingTools(true); - const timer = setTimeout(() => fetchTools(() => active), toolPreview.kind === "preview" ? 500 : 0); + const timer = setTimeout( + () => fetchTools(() => !controller.signal.aborted), + toolPreview.kind === "preview" ? 500 : 0, + ); return () => { - active = false; + controller.abort(); clearTimeout(timer); }; // eslint-disable-next-line react-hooks/exhaustive-deps From 7c1d060b7af3150320a2c88428871703d71f7a38 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:04:36 -0700 Subject: [PATCH 267/411] fix(mcp): enforce OAuth identity binding across credential lifetime --- litellm/proxy/_experimental/mcp_server/db.py | 34 +++- .../mcp_server/discoverable_endpoints.py | 94 +++++++--- .../mcp_server/oauth_identity_binding.py | 170 ++++++++++++++---- .../authz_code_refresher.py | 37 +++- .../per_user_oauth_store.py | 13 ++ .../types/mcp_server/mcp_server_manager.py | 13 +- .../test_authz_code_refresher.py | 34 ++++ .../test_per_user_oauth_store.py | 25 +++ .../mcp_server/test_db_credentials.py | 24 ++- .../mcp_server/test_discoverable_endpoints.py | 60 ++++++- .../mcp_server/test_oauth_identity_binding.py | 111 +++++++++--- .../test_mcp_management_endpoints.py | 3 +- 12 files changed, 526 insertions(+), 92 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 7379126983a..08f83aeb756 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -6,10 +6,17 @@ from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, cast +from typing_extensions import ReadOnly + from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + RefreshTokenPresented, + credential_binding_matches, + enforce_oauth_identity_binding, +) from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request from litellm.proxy._types import ( LiteLLM_MCPServerTable, @@ -117,6 +124,7 @@ class _OAuthCredentialAccessToken(TypedDict): class OAuthCredentialPayload(_OAuthCredentialAccessToken, total=False): + identity_binding_proof: ReadOnly[str] type: str refresh_token: str expires_at: str @@ -1393,6 +1401,7 @@ async def store_user_oauth_credential( expires_in: int | None = None, scopes: list[str] | None = None, skip_byok_guard: bool = False, + identity_binding_proof: str | None = None, ) -> None: """Persist an OAuth2 access token for a user+server pair. @@ -1409,6 +1418,7 @@ async def store_user_oauth_credential( "type": "oauth2", "access_token": access_token, "connected_at": datetime.now(timezone.utc).isoformat(), + **({"identity_binding_proof": identity_binding_proof} if identity_binding_proof else {}), } if refresh_token: payload["refresh_token"] = refresh_token @@ -1628,6 +1638,11 @@ async def refresh_user_oauth_token( warning and returns ``None`` — the caller is responsible for clearing the stale credential and triggering re-authentication. """ + binding: Final = server.oauth_identity_binding + if binding is not None and binding.mode == "enforce": + if not await credential_binding_matches(binding, user_id, server.server_id, cred): + return None + refresh_token: Final[str | None] = cred.get("refresh_token") token_url: Final[str | None] = getattr(server, "effective_token_url", None) or getattr(server, "token_url", None) server_id: Final[str] = getattr(server, "server_id", "") @@ -1677,6 +1692,14 @@ async def refresh_user_oauth_token( ) return None + binding_proof: Final = await enforce_oauth_identity_binding( + server=server, + token_response=body, + litellm_user_id=user_id, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented(refresh_token), + ) + access_token: Final[str | None] = body.get("access_token") if not access_token: verbose_proxy_logger.warning( @@ -1709,6 +1732,7 @@ async def refresh_user_oauth_token( refresh_token=new_refresh_token, expires_in=expires_in, scopes=scopes, + identity_binding_proof=binding_proof, skip_byok_guard=True, # Row is already OAuth2; skip the extra find_unique check ) @@ -1742,6 +1766,10 @@ async def resolve_valid_user_oauth_token( grant: Final = oauth_grant_state(cred) if cred is None or grant == "absent": return None + binding: Final = server.oauth_identity_binding + if binding is not None and binding.mode == "enforce": + if not await credential_binding_matches(binding, user_id, server.server_id, cred): + return None if grant == "valid": return cred if prisma_client is None: @@ -1782,7 +1810,11 @@ async def resolve_user_oauth_access_token( mcp_per_user_token_cache, ) - if prefetched_creds is None: + binding: Final = server.oauth_identity_binding + enforce_binding: Final = binding is not None and binding.mode == "enforce" + if enforce_binding: + await mcp_per_user_token_cache.delete(user_id, server_id) + if prefetched_creds is None and not enforce_binding: cached_token: Final = await mcp_per_user_token_cache.get(user_id, server_id) if cached_token is not None: return cached_token diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 9f6bc1860e1..c8e02be308b 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -144,6 +144,7 @@ def encode_state_with_base_url( dcr_client_id: str | None = None, dcr_client_secret: str | None = None, dcr_token_endpoint_auth_method: MCPTokenEndpointAuthMethod | None = None, + oauth_nonce: str | None = None, ) -> str: """ Encode the base_url, original state, and PKCE parameters using encryption. @@ -154,9 +155,8 @@ def encode_state_with_base_url( code_challenge: PKCE code challenge from client code_challenge_method: PKCE code challenge method from client client_redirect_uri: Original redirect_uri from client - litellm_user_id: The SSO-authenticated litellm user captured at the bridge authorize - (interactive dcr_bridge oauth_delegate only); the callback seals it into the gateway - authorization code so the token mint can bind the envelope to this user + litellm_user_id: The authenticated user captured for bridge or identity-bound per-user OAuth; + the callback seals this credential owner into the authorization code mcp_server_id: The server the flow targets, sealed alongside litellm_user_id (bridge) or dcr_client_id (ephemeral mint) so the gateway code cannot be replayed against another server @@ -174,6 +174,7 @@ def encode_state_with_base_url( An encrypted string that encodes all values """ state_data: Final = { + "oauth_nonce": oauth_nonce, "base_url": base_url, "original_state": original_state, "code_challenge": code_challenge, @@ -215,10 +216,10 @@ _BRIDGE_AUTH_CODE_PREFIX: Final = "llm_bcode_" class _BridgeAuthorizationCode(BaseModel): - """The identity and upstream code the gateway seals into the authorization code it hands a DCR - client for an interactive dcr_bridge oauth_delegate sign-in, recovered at the token endpoint.""" + """Authenticated caller and upstream code sealed for bridge or identity-bound per-user OAuth.""" model_config = ConfigDict(frozen=True) + oauth_nonce: str | None = None upstream_code: str = Field(min_length=1) litellm_user_id: str = Field(min_length=1) mcp_server_id: str = Field(min_length=1) @@ -230,7 +231,12 @@ def is_bridge_authorization_code(code: str) -> bool: return code.startswith(_BRIDGE_AUTH_CODE_PREFIX) -def seal_bridge_authorization_code(upstream_code: str, litellm_user_id: str, mcp_server_id: str) -> str: +def seal_bridge_authorization_code( + upstream_code: str, + litellm_user_id: str, + mcp_server_id: str, + oauth_nonce: str | None = None, +) -> str: """Seal the upstream authorization code and the SSO-captured litellm user into a gateway authorization code. The DCR client only echoes this opaque value back at the token endpoint; the gateway decrypts it there to recover the user (to bind the envelope) and the upstream code (to @@ -239,7 +245,12 @@ def seal_bridge_authorization_code(upstream_code: str, litellm_user_id: str, mcp authenticated symmetric helper (the same family the OAuth state uses), so the client can neither read nor forge it.""" payload: Final = json.dumps( - {"upstream_code": upstream_code, "litellm_user_id": litellm_user_id, "mcp_server_id": mcp_server_id}, + { + "upstream_code": upstream_code, + "litellm_user_id": litellm_user_id, + "mcp_server_id": mcp_server_id, + "oauth_nonce": oauth_nonce, + }, sort_keys=True, ) return _BRIDGE_AUTH_CODE_PREFIX + encrypt_value_helper(payload) @@ -552,6 +563,7 @@ async def _store_per_user_token_server_side( server: MCPServer, user_id: str, token_response: dict[str, Any], + identity_binding_proof: str | None = None, ) -> None: """Persist the OAuth token server-side and warm the Redis cache. @@ -593,6 +605,7 @@ async def _store_per_user_token_server_side( refresh_token=refresh_token, expires_in=expires_in, scopes=scopes, + identity_binding_proof=identity_binding_proof, ) verbose_logger.info( "_store_per_user_token_server_side: stored token for user=%s server=%s", @@ -859,6 +872,11 @@ async def authorize_with_server( ), ) + binding: Final = resolved_server.oauth_identity_binding + enforce_binding: Final = binding is not None and binding.mode == "enforce" + if enforce_binding: + _require_s256_pkce(code_challenge, code_challenge_method) + if resolved_server.is_dcr_bridge: # Enforce S256 PKCE on both bridge arms. The relay arm forwards the validated, # now-non-optional pair to the upstream authorize; the short-circuit arm keeps @@ -889,19 +907,16 @@ async def authorize_with_server( base_url: Final = urlunparse(parsed._replace(query="")) request_base_url: Final = get_request_base_url(request) - # Interactive dcr_bridge oauth_delegate sign-in: this arm runs the gateway /callback and /token in - # the loop, so the gateway can capture the litellm user here (from the browser's UI session) and - # carry it to the back-channel token mint. Seal the SSO user and the target server into the state; - # the callback reads them back to mint the gateway authorization code. A DCR client cannot present a - # litellm key, so the browser session is the only identity source; without one there is nothing to - # bind, so send the user through login first. Every other oauth2 server keeps the identity-less state. + # Seal the authenticated caller into state so the token exchange cannot select another credential owner. litellm_user_id: str | None = None - if resolved_server.is_dcr_bridge and resolved_server.is_oauth_delegate: + if enforce_binding or (resolved_server.is_dcr_bridge and resolved_server.is_oauth_delegate): from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # inline import avoids a module-load circular import _user_id_from_session_cookie, ) - litellm_user_id = _user_id_from_session_cookie(request) + litellm_user_id = ( + await _extract_user_id_from_request(request) if enforce_binding else None + ) or _user_id_from_session_cookie(request) if litellm_user_id is None: return _redirect_to_litellm_login(request) denial: Final = await _bridge_authorize_access_denial( @@ -913,9 +928,11 @@ async def authorize_with_server( if denial is not None: return denial + oauth_nonce: Final = secrets.token_urlsafe(32) if enforce_binding else None encoded_state: Final = encode_state_with_base_url( base_url=base_url, original_state=state, + oauth_nonce=oauth_nonce, code_challenge=code_challenge, code_challenge_method=code_challenge_method, client_redirect_uri=redirect_uri, @@ -935,11 +952,16 @@ async def authorize_with_server( "state": relay_state, "response_type": response_type or "code", } + if oauth_nonce: + params["nonce"] = oauth_nonce if scope: params["scope"] = scope elif resolved_server.scopes: params["scope"] = " ".join(resolved_server.scopes) + if enforce_binding and "openid" not in params.get("scope", "").split(): + params["scope"] = f"openid {params.get('scope', '')}".strip() + if code_challenge: params["code_challenge"] = code_challenge if code_challenge_method: @@ -1020,6 +1042,12 @@ async def exchange_token_with_server( except TokenEndpointAuthConfigError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc + request_user_id: Final = ( + await _extract_user_id_from_request(request) + if resolved_server.needs_user_oauth_token or resolved_server.oauth_identity_binding is not None + else None + ) + bridge_identity: _BridgeAuthorizationCode | None = None bridge_mint_ready: _BridgeMintReady | None = None bridge_upstream_refresh: SecretStr | None = None @@ -1081,6 +1109,14 @@ async def exchange_token_with_server( detail="Authorization code was issued for a different MCP server", ) code = bridge_identity.upstream_code + binding: Final = resolved_server.oauth_identity_binding + if binding is not None and binding.mode == "enforce": + if bridge_identity is None or not bridge_identity.oauth_nonce: + raise HTTPException(status_code=403, detail={"error": "oauth_identity_binding_failed"}) + if request_user_id is not None and request_user_id != bridge_identity.litellm_user_id: + raise HTTPException(status_code=403, detail={"error": "oauth_principal_mismatch"}) + if not code_verifier: + raise HTTPException(status_code=403, detail={"error": "oauth_identity_binding_failed"}) bridge_token_relay: Final = _dcr_bridge_relays_client_registration(resolved_server) if bridge_token_relay and not redirect_uri: raise HTTPException( @@ -1108,6 +1144,16 @@ async def exchange_token_with_server( return _bridge_mint_error_response(prepared) bridge_mint_ready = prepared + refresh_binding: Final = resolved_server.oauth_identity_binding + if grant_type == "refresh_token" and refresh_binding is not None and refresh_binding.mode == "enforce": + await enforce_oauth_identity_binding( + server=resolved_server, + token_response={}, + litellm_user_id=request_user_id, + grant_type=grant_type, + refresh_ownership=refresh_ownership, + ) + async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) try: response: Final = await async_client.post( @@ -1150,19 +1196,19 @@ async def exchange_token_with_server( # Bind the exchanged token to the LiteLLM caller BEFORE it is returned, stored, or cached, so a # token minted for a different upstream principal never becomes usable under the caller's user_id. - resolved_user_id: Final = ( - await _extract_user_id_from_request(request) - if resolved_server.needs_user_oauth_token or resolved_server.oauth_identity_binding is not None - else None - ) - if isinstance(token_response, dict): + resolved_user_id: Final = bridge_identity.litellm_user_id if bridge_identity else request_user_id + binding_proof: Final = ( await enforce_oauth_identity_binding( server=resolved_server, token_response=token_response, litellm_user_id=resolved_user_id, grant_type=grant_type, refresh_ownership=refresh_ownership, + expected_nonce=bridge_identity.oauth_nonce if bridge_identity else None, ) + if isinstance(token_response, dict) + else None + ) # Store server-side when the server is configured for per-user OAuth and # the calling client has provided a valid LiteLLM identity. @@ -1175,6 +1221,7 @@ async def exchange_token_with_server( server=resolved_server, user_id=user_id, token_response=token_response, + identity_binding_proof=binding_proof, ) except Exception as exc: verbose_logger.warning( @@ -2161,7 +2208,10 @@ async def callback( forwarded_code = code if isinstance(litellm_user_id, str) and litellm_user_id and isinstance(mcp_server_id, str) and mcp_server_id: forwarded_code = seal_bridge_authorization_code( - upstream_code=code, litellm_user_id=litellm_user_id, mcp_server_id=mcp_server_id + upstream_code=code, + litellm_user_id=litellm_user_id, + mcp_server_id=mcp_server_id, + oauth_nonce=state_data.get("oauth_nonce"), ) elif isinstance(dcr_client_id, str) and dcr_client_id and isinstance(mcp_server_id, str) and mcp_server_id: forwarded_code = seal_passthrough_authorization_code( diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index 58ec3cfdb4f..7ca9e6a23df 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -8,9 +8,12 @@ claim is compared to the caller's trusted LiteLLM identity. Mismatches fail clos and are logged in audit mode. """ +import hashlib +import hmac +import json from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass -from typing import Final, Literal, TypeAlias +from typing import Final, Literal, Protocol, TypeAlias import jwt from fastapi import HTTPException @@ -45,9 +48,17 @@ CallerPrincipalLoader: TypeAlias = Callable[ [str, MCPOAuthIdentityBinding], # mutable-ok: Callable parameter syntax requires a list Awaitable[str | None], ] + + +@dataclass(frozen=True, slots=True) +class VerifiedRefreshToken: + refresh_token: str + binding_proof: str + + StoredRefreshTokenLoader: TypeAlias = Callable[ - [str, str], # mutable-ok: Callable parameter syntax requires a list - Awaitable[str | None], + [str, str, MCPOAuthIdentityBinding], # mutable-ok: Callable parameter syntax requires a list + Awaitable[VerifiedRefreshToken | None], ] _RejectionCode: TypeAlias = Literal["oauth_principal_mismatch", "oauth_identity_binding_failed"] @@ -72,6 +83,18 @@ class RefreshTokenPresented: RefreshOwnership: TypeAlias = RefreshOwnershipProven | RefreshTokenPresented | None +class BindingValidator(Protocol): + async def __call__( + self, + *, + server: MCPServer, + token_response: Mapping[str, object], + litellm_user_id: str | None, + grant_type: str, + refresh_ownership: RefreshOwnership, + ) -> str | None: ... + + async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mapping[str, object]]: jwks_url: Final[str] = binding.jwks_url or await _discover_jwks_url(binding.issuer) cached: Final = await _jwks_cache.async_get_cache(jwks_url) @@ -118,7 +141,7 @@ def _decode_id_token( signing_key: "jwt.PyJWK", ) -> "Mapping[str, object] | _BindingRejection": try: - decode_options: Final[Options] = {"require": ("iss", "exp")} + decode_options: Final[Options] = {"require": ("iss", "exp", "aud", "sub", "iat")} return jwt.decode( id_token, signing_key.key, @@ -169,7 +192,9 @@ async def _load_caller_principal(litellm_user_id: str, binding: MCPOAuthIdentity return loaded.user_email -async def _load_stored_refresh_token(litellm_user_id: str, server_id: str) -> str | None: +async def _load_stored_refresh_token( + litellm_user_id: str, server_id: str, binding: MCPOAuthIdentityBinding +) -> VerifiedRefreshToken | None: try: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # keep database imports lazy get_user_oauth_credential, @@ -184,17 +209,89 @@ async def _load_stored_refresh_token(litellm_user_id: str, server_id: str) -> st user_id=litellm_user_id, server_id=server_id, ) - return cred.get("refresh_token") if cred else None + if not cred or not await credential_binding_matches(binding, litellm_user_id, server_id, cred): + return None + refresh_token: Final = cred.get("refresh_token") + proof: Final = cred.get("identity_binding_proof") + return VerifiedRefreshToken(refresh_token, proof) if refresh_token and proof else None except Exception: # noqa: BLE001 # a credential lookup failure must fail closed return None +async def current_binding_proof( + binding: MCPOAuthIdentityBinding, + user_id: str, + server_id: str, + caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, +) -> str | None: + principal: Final = await caller_principal_loader(user_id, binding) + if not principal: + return None + return _binding_proof(binding, user_id, server_id, principal) + + +def _binding_proof(binding: MCPOAuthIdentityBinding, user_id: str, server_id: str, principal: str) -> str: + payload: Final = json.dumps( + ("oidc-nonce-v1", server_id, user_id, principal, binding.model_dump(mode="json")), + sort_keys=True, + ) + return hashlib.sha256(payload.encode()).hexdigest() + + +async def credential_binding_matches( + binding: MCPOAuthIdentityBinding, + user_id: str, + server_id: str, + credential: Mapping[str, object], + caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, +) -> bool: + stored: Final = credential.get("identity_binding_proof") + if not isinstance(stored, str) or not stored: + return False + expected: Final = await current_binding_proof(binding, user_id, server_id, caller_principal_loader) + return expected is not None and hmac.compare_digest(stored, expected) + + def _principals_match(upstream: str, caller: str, binding: MCPOAuthIdentityBinding) -> bool: if binding.principal_claim == "email" or binding.caller_field == "user_email": return upstream.strip().casefold() == caller.strip().casefold() return upstream == caller +async def _evaluate_refresh_ownership( + binding: MCPOAuthIdentityBinding, + litellm_user_id: str | None, + server_id: str, + refresh_ownership: RefreshOwnership, + stored_refresh_token_loader: StoredRefreshTokenLoader, +) -> _BindingRejection | str: + match refresh_ownership: + case RefreshOwnershipProven(): + return _BindingRejection( + code="oauth_identity_binding_failed", + description="an identity envelope alone does not prove upstream principal binding", + ) + case None: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="refresh_token grant without an id_token carries no refresh token to prove ownership of", + ) + case RefreshTokenPresented(refresh_token): + if not litellm_user_id: + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the request carries no resolvable LiteLLM user identity to bind the credential to", + ) + stored: Final = await stored_refresh_token_loader(litellm_user_id, server_id, binding) + if stored is None or not hmac.compare_digest(stored.refresh_token, refresh_token): + return _BindingRejection( + code="oauth_identity_binding_failed", + description="the presented refresh_token is not the caller's stored credential for this server", + ) + return stored.binding_proof + assert_never(refresh_ownership) # pragma: no cover + + async def _evaluate_binding( binding: MCPOAuthIdentityBinding, token_response: Mapping[str, object], @@ -205,7 +302,8 @@ async def _evaluate_binding( jwks_fetcher: JwksFetcher, caller_principal_loader: CallerPrincipalLoader, stored_refresh_token_loader: StoredRefreshTokenLoader, -) -> _BindingRejection | None: + expected_nonce: str | None, +) -> _BindingRejection | str: id_token: Final = token_response.get("id_token") if not isinstance(id_token, str) or not id_token: if grant_type != "refresh_token": @@ -213,28 +311,13 @@ async def _evaluate_binding( code="oauth_identity_binding_failed", description="the upstream token response carries no id_token to bind the credential to a principal", ) - match refresh_ownership: - case RefreshOwnershipProven(): - return None - case None: - return _BindingRejection( - code="oauth_identity_binding_failed", - description="refresh_token grant without an id_token carries no refresh token to prove ownership of", - ) - case RefreshTokenPresented(refresh_token): - if not litellm_user_id: - return _BindingRejection( - code="oauth_identity_binding_failed", - description="the request carries no resolvable LiteLLM user identity to bind the credential to", - ) - stored: Final = await stored_refresh_token_loader(litellm_user_id, server_id) - if stored is None or stored != refresh_token: - return _BindingRejection( - code="oauth_identity_binding_failed", - description="the presented refresh_token is not the caller's stored credential for this server", - ) - return None - assert_never(refresh_ownership) # pragma: no cover + return await _evaluate_refresh_ownership( + binding, + litellm_user_id, + server_id, + refresh_ownership, + stored_refresh_token_loader, + ) if not litellm_user_id: return _BindingRejection( code="oauth_identity_binding_failed", @@ -247,12 +330,25 @@ async def _evaluate_binding( code="oauth_identity_binding_failed", description=f"could not fetch the issuer's JWKS: {exc}", ) - signing_key: Final = _select_signing_key(id_token, keys) + try: + signing_key: Final = _select_signing_key(id_token, keys) + except (jwt.PyJWTError, ValueError, TypeError): + return _BindingRejection( + code="oauth_identity_binding_failed", + description="invalid id_token header or issuer signing key", + ) if isinstance(signing_key, _BindingRejection): return signing_key claims: Final = _decode_id_token(id_token, binding, signing_key) if isinstance(claims, _BindingRejection): return claims + if grant_type == "authorization_code": + nonce: Final = claims.get("nonce") + if not expected_nonce or not isinstance(nonce, str) or not hmac.compare_digest(nonce, expected_nonce): + return _BindingRejection( + code="oauth_identity_binding_failed", + description="id_token nonce does not match the authenticated authorization transaction", + ) upstream: Final = _upstream_principal(claims, binding) if isinstance(upstream, _BindingRejection): return upstream @@ -267,7 +363,7 @@ async def _evaluate_binding( code="oauth_principal_mismatch", description="The browser account does not match the selected credential owner.", ) - return None + return _binding_proof(binding, litellm_user_id, server_id, caller) async def enforce_oauth_identity_binding( @@ -279,16 +375,17 @@ async def enforce_oauth_identity_binding( jwks_fetcher: JwksFetcher = _fetch_issuer_jwks, caller_principal_loader: CallerPrincipalLoader = _load_caller_principal, stored_refresh_token_loader: StoredRefreshTokenLoader = _load_stored_refresh_token, -) -> None: + expected_nonce: str | None = None, +) -> str | None: """Validate the exchanged token's upstream principal against the LiteLLM caller. No-op when the server has no binding or it is disabled. In enforce mode a failure raises 403 before the caller returns, stores, or caches the token; in audit mode failures are logged only. A refresh_token grant without an id_token is allowed only when the presented refresh token matches - the caller's stored credential or a sealed bridge envelope already proved ownership. + the caller's stored credential and that credential was previously identity-validated. """ binding: Final = server.oauth_identity_binding - if binding is None or binding.mode == "disabled": + if binding is None or binding.mode not in ("audit", "enforce"): return rejection: Final = await _evaluate_binding( binding=binding, @@ -300,9 +397,10 @@ async def enforce_oauth_identity_binding( jwks_fetcher=jwks_fetcher, caller_principal_loader=caller_principal_loader, stored_refresh_token_loader=stored_refresh_token_loader, + expected_nonce=expected_nonce, ) - if rejection is None: - return + if isinstance(rejection, str): + return rejection if binding.mode == "enforce" else None if binding.mode == "audit": verbose_logger.warning( "oauth_identity_binding audit: server=%s user=%s grant=%s rejected=%s (%s)", diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py index 92bd30694af..824459fd9b9 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py @@ -18,6 +18,11 @@ from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( TokenEndpointAuthConfigError, ) +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( + BindingValidator, + RefreshTokenPresented, + enforce_oauth_identity_binding, +) from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, @@ -39,6 +44,7 @@ class CredentialPersist(Protocol): refresh_token: str | None, expires_in: int | None, scopes: tuple[str, ...] | None, + identity_binding_proof: str | None = None, ) -> None: ... @@ -78,11 +84,13 @@ class AuthorizationCodeRefresher: persist: CredentialPersist, *, clock: Callable[[], float] = time.time, + identity_validator: BindingValidator = enforce_oauth_identity_binding, ) -> None: self._server_lookup = server_lookup self._token_endpoint = token_endpoint self._persist = persist self._clock = clock + self._identity_validator = identity_validator async def refresh(self, user_id: str, server_id: str, token: OAuthToken) -> OAuthToken | None: if token.refresh_token is None: @@ -104,6 +112,15 @@ class AuthorizationCodeRefresher: except TokenEndpointAuthConfigError as exc: verbose_logger.warning("MCP OAuth refresh misconfigured for server %s: %s", server_id, exc) return None + binding: Final = server.oauth_identity_binding + if binding is not None and binding.mode == "enforce": + await self._identity_validator( + server=server, + token_response={}, + litellm_user_id=user_id, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented(token.refresh_token), + ) form: Final = { "grant_type": "refresh_token", "refresh_token": token.refresh_token, @@ -116,12 +133,30 @@ class AuthorizationCodeRefresher: if not isinstance(access_token, str) or not access_token: return None + binding_proof: Final = await self._identity_validator( + server=server, + token_response=body, + litellm_user_id=user_id, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented(token.refresh_token), + ) rotated: Final = body.get("refresh_token") new_refresh: Final = rotated if isinstance(rotated, str) and rotated else token.refresh_token expires_in: Final = _parse_expires_in(body.get("expires_in")) scopes: Final = _parse_scopes(body.get("scope")) or token.scopes - await self._persist(user_id, server_id, access_token, new_refresh, expires_in, scopes or None) + if binding_proof is not None: + await self._persist( + user_id, + server_id, + access_token, + new_refresh, + expires_in, + scopes or None, + identity_binding_proof=binding_proof, + ) + else: + await self._persist(user_id, server_id, access_token, new_refresh, expires_in, scopes or None) return OAuthToken( access_token=access_token, expires_at=self._clock() + expires_in if expires_in is not None else None, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py index 5e28396dcfb..efcc613c539 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py @@ -15,6 +15,7 @@ from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger +from litellm.proxy._experimental.mcp_server.oauth_identity_binding import credential_binding_matches from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import ( AuthorizationCodeRefresher, ) @@ -38,6 +39,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_cod OAuthTokenCacheCodec, ) from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import ( + CredentialReader, V2PerUserTokenStore, ) @@ -69,6 +71,7 @@ async def _persist_credential( refresh_token: str | None, expires_in: int | None, scopes: tuple[str, ...] | None, + identity_binding_proof: str | None = None, ) -> None: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 store_user_oauth_credential, @@ -86,6 +89,7 @@ async def _persist_credential( expires_in=expires_in, scopes=list(scopes) if scopes else None, skip_byok_guard=True, + identity_binding_proof=identity_binding_proof, ) @@ -172,8 +176,10 @@ class LazyPerUserOAuthTokenStore: *, store_builder: StoreBuilder = _build_per_user_oauth_token_store, redis_available: Callable[[], bool] = _redis_cache_is_available, + credential_reader: CredentialReader = _read_credential, ) -> None: self._server_lookup = server_lookup + self._credential_reader = credential_reader self._store_builder = store_builder self._redis_available = redis_available self._store: InvalidatableOAuthTokenStore | None = None @@ -182,6 +188,13 @@ class LazyPerUserOAuthTokenStore: self._local_fetches = 0 async def fetch(self, user_id: str, server_id: str) -> OAuthToken | None: + server: Final = self._server_lookup(server_id) + binding: Final = server.oauth_identity_binding if server else None + if binding is not None and binding.mode == "enforce": + credential: Final = await self._credential_reader(user_id, server_id) + await self.invalidate(user_id, server_id) + if credential is None or not await credential_binding_matches(binding, user_id, server_id, credential): + return None if self._uses_redis: store = self._store if store is not None: diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 117d95a09de..f6b5441a626 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -1,7 +1,8 @@ from datetime import datetime from typing import Any, Final, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from typing_extensions import Self from litellm.types.mcp import ( DEFAULT_SUBJECT_TOKEN_TYPE, @@ -42,7 +43,7 @@ class MCPOAuthIdentityBinding(BaseModel): """Per-server policy binding stored per-user OAuth credentials to the authenticated LiteLLM caller. When enabled for an interactive oauth2 server, the token relay validates the upstream OIDC - ``id_token`` (signature via the pinned issuer's JWKS, issuer, audience, expiry) and compares its + ``id_token`` (signature via the pinned issuer's JWKS, issuer, audience, expiry, nonce) and compares its principal claim to the LiteLLM caller's trusted identity before the token is returned, stored, or cached. ``audit`` logs mismatches without changing behavior; ``enforce`` fails closed with 403 ``oauth_principal_mismatch`` and disables the direct ``oauth-user-credential`` POST, which @@ -246,6 +247,14 @@ class MCPServer(BaseModel): """ return self.oauth2_flow == "client_credentials" + @model_validator(mode="after") + def validate_identity_binding_mode(self) -> Self: + binding: Final = self.oauth_identity_binding + if binding is not None and binding.mode != "disabled": + if not self.needs_user_oauth_token or self.delegate_auth_to_upstream: + raise ValueError("oauth_identity_binding requires gateway-managed per-user OAuth2 credentials") + return self + @property def needs_user_oauth_token(self) -> bool: """True if this is an OAuth2 server that relies on per-user tokens (no client_credentials).""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py index bb2f2ff8b02..16707f6ace8 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py @@ -22,6 +22,7 @@ class _Server: server_id="srv", configured_token_url=None, ): + self.oauth_identity_binding = None self.token_url = token_url self.configured_token_url = configured_token_url self.client_id = client_id @@ -287,3 +288,36 @@ async def test_refresh_uses_admin_entered_token_url_when_issuer_yield_empties_re assert token is not None assert token.access_token == "new-at" assert posted[0][0] == "https://idp.example.com/token" + + +@pytest.mark.asyncio +async def test_identity_rejection_never_persists_or_returns_refreshed_token(): + from unittest.mock import AsyncMock + from fastapi import HTTPException + + validator = AsyncMock(side_effect=HTTPException(status_code=403, detail="oauth_principal_mismatch")) + persist = AsyncMock() + refresher = AuthorizationCodeRefresher( + _lookup(_Server()), _endpoint({"access_token": "foreign-token", "id_token": "foreign-identity"}), + persist, identity_validator=validator, + ) + with pytest.raises(HTTPException) as error: + await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) + assert error.value.status_code == 403 + persist.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_verified_refresh_preserves_binding_proof_in_storage(): + from unittest.mock import AsyncMock + + validator = AsyncMock(return_value="verified-binding") + persist = AsyncMock() + refresher = AuthorizationCodeRefresher( + _lookup(_Server()), _endpoint({"access_token": "new", "refresh_token": "rotated"}), + persist, identity_validator=validator, + ) + token = await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) + assert token.access_token == "new" + assert token.refresh_token == "rotated" + assert persist.await_args.kwargs["identity_binding_proof"] == "verified-binding" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py index ca32cf2bb8d..bd190b09c82 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py @@ -246,3 +246,28 @@ async def test_lazy_store_invalidate_works_after_redis_chain_is_built() -> None: assert build_calls == 1 assert redis_store.invalidations == [("u", "s")] + + +@pytest.mark.asyncio +async def test_enforcement_invalidates_cached_legacy_credentials_before_use(): + from litellm.types.mcp import MCPAuth, MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + server = MCPServer( + server_id="srv", name="srv", transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", issuer="https://idp.example.com", audiences=["client"], + ), + ) + cached = _RecordingStore("belongs-to-bob") + + async def read_legacy(user_id, server_id): + return {"access_token": "belongs-to-bob", "refresh_token": "bobs-refresh"} + + store = LazyPerUserOAuthTokenStore( + lambda server_id: server, store_builder=lambda lookup: (cached, False), + redis_available=lambda: False, credential_reader=read_legacy, + ) + assert await store.fetch("alice", "srv") is None + assert cached.calls == [] + assert cached.invalidations == [("alice", "srv")] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 355b3bfd30e..8e2adbeb06e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -628,6 +628,7 @@ async def test_oauth_round_trip_returns_payload(): access_token, refresh_token="rfr-xyz", scopes=["a", "b"], + identity_binding_proof="verified-proof", ) stored = _stored_value(prisma) @@ -642,6 +643,7 @@ async def test_oauth_round_trip_returns_payload(): assert result["access_token"] == access_token assert result["refresh_token"] == "rfr-xyz" assert result["scopes"] == ["a", "b"] + assert result["identity_binding_proof"] == "verified-proof" @pytest.mark.asyncio @@ -1184,6 +1186,7 @@ class _RefreshResponse: def _refresh_server(**overrides): base = dict( + oauth_identity_binding=None, token_url="https://idp.example.com/token", server_id="srv-1", client_id="cid", @@ -1309,7 +1312,7 @@ async def test_refresh_user_oauth_token_uses_client_secret_basic(monkeypatch): sends HTTP Basic and keeps the secret out of the body.""" import litellm.proxy._experimental.mcp_server.db as db_mod - server = MagicMock() + server = MagicMock(oauth_identity_binding=None) server.token_url = "https://idp.example.com/oauth2/token" server.server_id = "srv" server.client_id = "cid" @@ -1348,7 +1351,7 @@ async def test_refresh_user_oauth_token_defaults_to_client_secret_post(monkeypat the body (client_secret_post) and sends no Authorization header.""" import litellm.proxy._experimental.mcp_server.db as db_mod - server = MagicMock() + server = MagicMock(oauth_identity_binding=None) server.token_url = "https://idp.example.com/oauth2/token" server.server_id = "srv" server.client_id = "cid" @@ -1609,3 +1612,20 @@ def test_partial_update_defers_omitted_eligibility_fields_to_the_stored_row(): request = UpdateMCPServerRequest(server_id="relay-update", per_server_oauth_discovery=True) assert request.per_server_oauth_discovery is True + + +@pytest.mark.asyncio +async def test_enforcement_rejects_preexisting_unverified_credential(): + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + server = MCPServer( + server_id="srv-1", name="srv-1", url="https://mcp.example.com", transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", issuer="https://idp.example.com", audiences=["client"], + ), + ) + result = await resolve_valid_user_oauth_token( + user_id="alice", server=server, + cred={"access_token": "belongs-to-bob", "refresh_token": "bobs-refresh-token"}, + ) + assert result is None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 590d6606154..42187cb1771 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -8194,15 +8194,17 @@ async def test_token_exchange_refresh_passes_presented_refresh_ownership(): @pytest.mark.asyncio -async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): +async def test_token_exchange_authorization_code_passes_no_refresh_ownership(monkeypatch): from fastapi import Request from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( exchange_token_with_server, + seal_bridge_authorization_code, ) from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + monkeypatch.setenv("LITELLM_SALT_KEY", "identity-binding-test-salt") server = MCPServer( server_id="srv-1", name="srv-1", @@ -8244,16 +8246,17 @@ async def test_token_exchange_authorization_code_passes_no_refresh_ownership(): request=request, mcp_server=server, grant_type="authorization_code", - code="auth-code", + code=seal_bridge_authorization_code("auth-code", "user-a", "srv-1", "login-nonce"), redirect_uri="https://litellm.example.com/callback", client_id="cid", client_secret=None, - code_verifier=None, + code_verifier="test-verifier", ) assert result.status_code == 200 assert json.loads(result.body)["access_token"] == "at" assert enforce.await_args.kwargs["refresh_ownership"] is None + assert enforce.await_args.kwargs["expected_nonce"] == "login-nonce" def _upstream_token_response(status_code: int, *, json_body: object = None, text_body: str = "") -> "httpx.Response": @@ -11049,3 +11052,54 @@ def test_introspect_route_answers_for_authenticated_caller(monkeypatch): assert active.status_code == 200 assert active.json()["active"] is True assert active.json()["sub"] == "u1" + + +@pytest.mark.asyncio +async def test_identity_bound_authorization_carries_nonce_and_caller_through_callback(monkeypatch): + from http.cookies import SimpleCookie + from urllib.parse import parse_qs, urlparse + from fastapi import Request + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _oauth_state_cookie_name, authorize_with_server, callback, open_bridge_authorization_code, + ) + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + monkeypatch.setenv("LITELLM_SALT_KEY", "identity-binding-test-salt") + server = MCPServer( + server_id="srv", name="srv", transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + client_id="client", authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", issuer="https://idp.example.com", audiences=["client"], + ), + ) + request = Request({"type": "http", "scheme": "https", "server": ("proxy.example.com", 443), + "path": "/authorize", "query_string": b"", "headers": []}) + with ( + patch( # test-quality-ok: isolate authenticated request resolution from the real encrypted OAuth round trip + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", + new=AsyncMock(return_value="alice")), + patch( # test-quality-ok: isolate user access lookup while testing nonce and caller preservation + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._bridge_authorize_access_denial", + new=AsyncMock(return_value=None)), + ): + authorized = await authorize_with_server( + request, server, "client", "http://127.0.0.1:6274/callback", state="client-state", + code_challenge="pkce-challenge", code_challenge_method="S256", + ) + query = parse_qs(urlparse(authorized.headers["location"]).query) + assert len(query["nonce"][0]) >= 32 + cookies = SimpleCookie() + cookies.load(authorized.headers["set-cookie"]) + name = _oauth_state_cookie_name(query["state"][0]) + callback_request = Request({**request.scope, "path": "/callback", + "headers": [(b"cookie", f"{name}={cookies[name].value}".encode())]}) + completed = await callback(callback_request, code="upstream-code", state=query["state"][0]) + returned = parse_qs(urlparse(completed.headers["location"]).query) + sealed = open_bridge_authorization_code(returned["code"][0]) + assert sealed.litellm_user_id == "alice" + assert sealed.mcp_server_id == "srv" + assert sealed.upstream_code == "upstream-code" + assert sealed.oauth_nonce == query["nonce"][0] + assert returned["state"] == ["client-state"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index 36364c32ae5..6f291865e20 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -13,7 +13,9 @@ from pydantic import ValidationError from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( RefreshOwnershipProven, + VerifiedRefreshToken, RefreshTokenPresented, + current_binding_proof, _discover_jwks_url, _fetch_issuer_jwks, _load_caller_principal, @@ -21,7 +23,7 @@ from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( _select_signing_key, enforce_oauth_identity_binding, ) -from litellm.types.mcp import MCPTransport +from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer ISSUER: Final = "https://idp.example.com" @@ -48,6 +50,8 @@ def _sign_id_token(claims: Mapping[str, object]) -> str: "aud": AUDIENCE, "exp": int(time.time()) + 300, "iat": int(time.time()), + "nonce": "test-nonce", + "sub": "upstream-user", **claims, } return jwt.encode(payload, _PRIVATE_PEM, algorithm="RS256", headers={"kid": KID}) @@ -65,8 +69,8 @@ def _caller_loader(email: str | None): def _stored_refresh_token_loader(refresh_token: str | None): - async def load(_user_id: str, _server_id: str) -> str | None: - return refresh_token + async def load(_user_id: str, _server_id: str, _binding: MCPOAuthIdentityBinding) -> VerifiedRefreshToken | None: + return VerifiedRefreshToken(refresh_token, "verified-binding") if refresh_token else None return load @@ -77,6 +81,7 @@ def _server(mode: str = "enforce", **binding_overrides: object) -> MCPServer: name="srv-1", url="https://mcp.example.com", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth_identity_binding=MCPOAuthIdentityBinding( mode=mode, issuer=ISSUER, @@ -181,7 +186,9 @@ async def test_load_caller_principal_supports_user_id_and_database_email(): @pytest.mark.asyncio async def test_load_stored_refresh_token_returns_credential_and_fails_closed(): - get_credential: Final = AsyncMock(return_value={"refresh_token": "rt-1"}) + binding: Final = _server(caller_field="user_id", principal_claim="sub").oauth_identity_binding + proof: Final = await current_binding_proof(binding, "user-a", "srv-1") + get_credential: Final = AsyncMock(return_value={"refresh_token": "rt-1", "identity_binding_proof": proof}) with ( patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy._experimental.mcp_server.db.get_user_oauth_credential", @@ -192,13 +199,15 @@ async def test_load_stored_refresh_token_returns_credential_and_fails_closed(): return_value="prisma", ), ): - assert await _load_stored_refresh_token("user-a", "srv-1") == "rt-1" + assert await _load_stored_refresh_token("user-a", "srv-1", binding) == VerifiedRefreshToken("rt-1", proof) + get_credential.return_value = {"refresh_token": "rt-1"} + assert await _load_stored_refresh_token("user-a", "srv-1", binding) is None with patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection "litellm.proxy.utils.get_prisma_client_or_throw", side_effect=RuntimeError("database unavailable"), ): - assert await _load_stored_refresh_token("user-a", "srv-1") is None + assert await _load_stored_refresh_token("user-a", "srv-1", binding) is None @pytest.mark.asyncio @@ -209,11 +218,12 @@ async def test_matching_principal_passes(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) - assert result is None + assert result is not None @pytest.mark.asyncio @@ -225,6 +235,7 @@ async def test_mismatched_principal_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -243,6 +254,7 @@ async def test_missing_upstream_principal_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -263,6 +275,7 @@ async def test_jwks_fetch_failure_is_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=fail, caller_principal_loader=_caller_loader("alice@example.com"), @@ -284,6 +297,7 @@ async def test_missing_signing_key_is_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=no_keys, caller_principal_loader=_caller_loader("alice@example.com"), @@ -301,6 +315,7 @@ async def test_missing_caller_principal_is_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), @@ -334,11 +349,12 @@ async def test_user_id_principal_matching_uses_exact_comparison(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("user-a"), ) - assert result is None + assert result is not None @pytest.mark.asyncio @@ -349,6 +365,7 @@ async def test_missing_id_token_rejected_on_authorization_code(): token_response={"access_token": "at"}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -369,7 +386,7 @@ async def test_refresh_without_id_token_allowed_when_presented_token_matches_sto caller_principal_loader=_caller_loader("alice@example.com"), stored_refresh_token_loader=_stored_refresh_token_loader("rt-1"), ) - assert result is None + assert result is not None @pytest.mark.asyncio @@ -407,21 +424,18 @@ async def test_refresh_without_id_token_rejects_missing_stored_token(): @pytest.mark.asyncio -async def test_refresh_without_id_token_passes_when_bridge_proves_ownership(): - async def fail_if_called(_user_id: str, _server_id: str) -> str | None: - raise AssertionError("stored refresh token loader should not be called") - - result: Final = await enforce_oauth_identity_binding( - server=_server(), - token_response={"access_token": "at"}, - litellm_user_id="user-a", - grant_type="refresh_token", - refresh_ownership=RefreshOwnershipProven(), - jwks_fetcher=_jwks_fetcher, - caller_principal_loader=_caller_loader("alice@example.com"), - stored_refresh_token_loader=fail_if_called, - ) - assert result is None +async def test_identity_envelope_does_not_prove_upstream_binding(): + with pytest.raises(HTTPException) as error: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at"}, + litellm_user_id="user-a", + grant_type="refresh_token", + refresh_ownership=RefreshOwnershipProven(), + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert error.value.status_code == 403 @pytest.mark.asyncio @@ -464,6 +478,7 @@ async def test_audit_mode_logs_but_does_not_reject(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -480,6 +495,7 @@ async def test_unverified_email_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -503,6 +519,7 @@ async def test_wrong_issuer_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -519,6 +536,7 @@ async def test_no_litellm_identity_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id=None, grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), @@ -533,6 +551,7 @@ async def test_disabled_binding_is_noop(): token_response={"access_token": "at"}, litellm_user_id=None, grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), @@ -553,6 +572,7 @@ async def test_no_binding_is_noop(): token_response={"access_token": "at"}, litellm_user_id=None, grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader(None), @@ -576,9 +596,52 @@ async def test_wrong_audience_rejected(): token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", + expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) assert exc_info.value.status_code == 403 assert exc_info.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nonce", [None, "another-login"]) +async def test_authorization_code_rejects_missing_or_foreign_nonce(nonce): + token = _sign_id_token({"email": "alice@example.com", "email_verified": True, "nonce": nonce}) + with pytest.raises(HTTPException) as error: + await enforce_oauth_identity_binding( + server=_server(), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + expected_nonce="this-login", + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert error.value.status_code == 403 + assert error.value.detail["error"] == "oauth_identity_binding_failed" + + +@pytest.mark.asyncio +async def test_binding_proof_rejects_changed_user_or_policy(): + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import credential_binding_matches + + binding = _server(caller_field="user_id", principal_claim="sub").oauth_identity_binding + proof = await current_binding_proof(binding, "alice", "srv-1") + credential = {"identity_binding_proof": proof} + assert await credential_binding_matches(binding, "alice", "srv-1", credential) + assert not await credential_binding_matches(binding, "bob", "srv-1", credential) + assert not await credential_binding_matches(binding, "alice", "other-server", credential) + changed = binding.model_copy(update={"audiences": ["different-client"]}) + assert not await credential_binding_matches(changed, "alice", "srv-1", credential) + + +@pytest.mark.parametrize("auth_type", [MCPAuth.oauth_delegate, MCPAuth.true_passthrough]) +def test_identity_binding_rejects_modes_without_gateway_credential_custody(auth_type): + with pytest.raises(ValidationError, match="gateway-managed per-user"): + MCPServer( + server_id="srv", name="srv", transport=MCPTransport.http, auth_type=auth_type, + oauth_identity_binding=_server().oauth_identity_binding, + ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 6a20688c09d..5e00e7d75be 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -4807,7 +4807,7 @@ async def test_store_mcp_oauth_user_credential_blocked_when_identity_binding_enf """The direct opaque-token POST must be closed for enforce-mode identity-bound servers, otherwise it bypasses the token-relay principal check.""" from litellm.proxy._types import MCPOAuthUserCredentialRequest - from litellm.types.mcp import MCPTransport + from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer if not mgmt_endpoints.MCP_AVAILABLE: @@ -4824,6 +4824,7 @@ async def test_store_mcp_oauth_user_credential_blocked_when_identity_binding_enf name=server_id, url="https://mcp.example.com", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth_identity_binding=MCPOAuthIdentityBinding( mode="enforce", issuer="https://idp.example.com", From e34c4c8edc5fc6cd70fccf6a44c7765d920a6e28 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 18:09:28 -0700 Subject: [PATCH 268/411] fix(ui): scope shadow eval models to configured chat groups (#40488) Co-authored-by: Claude Code --- .../_components/ShadowEvalSection.test.tsx | 151 ++++++++++++++---- .../_components/ShadowEvalStartForm.tsx | 81 ++++------ .../hooks/models/useModels.test.ts | 103 +++++++++--- .../app/(dashboard)/hooks/models/useModels.ts | 61 ++++--- .../components/chat_ui/EndpointUtils.tsx | 35 +--- .../chat_ui/mode_endpoint_mapping.tsx | 17 ++ .../src/lib/autorouter_presets.test.ts | 13 ++ .../src/lib/autorouter_presets.ts | 13 +- 8 files changed, 317 insertions(+), 157 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index 64363da9933..1f73671caae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -1,3 +1,4 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { fireEvent, render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React from "react"; @@ -14,7 +15,9 @@ vi.mock("./useShadowEval", () => ({ })); const authorizedRoleMock = vi.fn(() => ({ accessToken: "token", isViewOnly: false })); -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => authorizedRoleMock() })); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ userId: "test-user-id", userRole: "Admin", ...authorizedRoleMock() }), +})); vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ useInfiniteKeys: vi.fn(() => ({ @@ -68,27 +71,33 @@ vi.mock("@/app/(dashboard)/hooks/users/useUsers", () => ({ })), })); -vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ +vi.mock("@/app/(dashboard)/hooks/models/useModels", async (importOriginal) => ({ + ...(await importOriginal()), useAutoRouters: vi.fn(() => ({ data: [ { model_name: "claude-auto", litellm_params: { model: "auto_router/claude-auto" } }, { model_name: "gpt-auto", litellm_params: { model: "auto_router/gpt-auto" } }, ], })), - usePlainModelGroups: vi.fn(() => new Set(["prod-claude"])), + usePlainModelGroups: vi.fn(() => new Set(["prod-claude", "prod-judge"])), + usePlainChatModelGroups: vi.fn(() => new Set(["prod-claude", "prod-judge"])), + usePlainChatModelDeployments: vi.fn(() => [ + { + model_name: "prod-judge", + litellm_params: { model: "anthropic/claude-sonnet-5" }, + model_info: { mode: "chat" }, + }, + ]), })); -vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ - useModelCostMap: vi.fn(() => ({ - data: { - "claude-sonnet-5": { litellm_provider: "anthropic", mode: "chat" }, - "gpt-4o": { litellm_provider: "openai", mode: "chat" }, - "gemini/gemini-2.5-pro": { litellm_provider: "gemini", mode: "chat" }, - "text-embedding-3-large": { litellm_provider: "openai", mode: "embedding" }, - }, - })), +vi.mock("@/components/networking", async (importOriginal) => ({ + ...(await importOriginal()), + modelInfoCall: vi.fn(), })); +import { usePlainChatModelGroups, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; +import { modelInfoCall } from "@/components/networking"; + import ShadowEvalSection, { shadowedTargetLabel } from "./ShadowEvalSection"; import { useShadowEvalJob, @@ -107,7 +116,7 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ models: [], direction: "forward", baseline_model: null, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", shadow_percentage: 10, targets: [ { @@ -249,6 +258,85 @@ describe("ShadowEvalSection", () => { if (defaultKeysImpl) vi.mocked(useInfiniteKeys).mockImplementation(defaultKeysImpl); }); + it("labels only configured judge recommendations", async () => { + const user = userEvent.setup(); + mockHooks({}); + render(); + + await user.click(screen.getByPlaceholderText("Select a judge model")); + expect(screen.getByRole("option", { name: /prod-judge.*Recommended/ })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: /openai\/gpt-4o/ })).not.toBeInTheDocument(); + + await user.keyboard("{Escape}"); + await chooseSelectOption( + user, + screen.getByText("Adoption check: key's traffic vs the router"), + "Regression check: router's picks vs a baseline", + ); + await user.click(screen.getByPlaceholderText("Select a baseline model")); + expect(screen.getByRole("option", { name: "prod-judge", exact: true })).toBeInTheDocument(); + expect(screen.queryByText("Recommended")).not.toBeInTheDocument(); + }); + + it("keeps custom models selectable through the real model hooks without widening chat choices to traffic filters", async () => { + const hooks = await vi.importActual( + "@/app/(dashboard)/hooks/models/useModels", + ); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const deployments = [ + { model_name: "custom-chat", litellm_params: { model: "openai/private-chat" } }, + { model_name: "custom-judge", litellm_params: { model: "openai/private-judge" }, model_info: { mode: null } }, + { + model_name: "embedding", + litellm_params: { model: "openai/private-embedding" }, + model_info: { mode: "embedding" }, + }, + { + model_name: "responses-only", + litellm_params: { model: "openai/private-responses" }, + model_info: { mode: "responses" }, + }, + { model_name: "auto-router", litellm_params: { model: "auto_router/complexity_router" } }, + ]; + vi.mocked(modelInfoCall).mockResolvedValue({ data: deployments, total_pages: 1 }); + const user = userEvent.setup(); + const { start } = mockHooks({}); + await vi.mocked(usePlainModelGroups).withImplementation(hooks.usePlainModelGroups, async () => { + await vi.mocked(usePlainChatModelGroups).withImplementation(hooks.usePlainChatModelGroups, async () => { + render( + + + , + ); + await chooseSelectOption(user, screen.getByPlaceholderText("Every model the targets use"), "responses-only"); + await chooseSelectOption(user, screen.getByPlaceholderText("Every model the targets use"), "custom-chat"); + await chooseSelectOption( + user, + screen.getByText("Adoption check: key's traffic vs the router"), + "Regression check: router's picks vs a baseline", + ); + await user.click(screen.getByPlaceholderText("Search keys by alias")); + await user.click(within(await screen.findByTestId("paginated-multi-select-list")).getByText("prod-alpha")); + await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); + await user.click(screen.getByPlaceholderText("Select a judge model")); + expect(screen.getAllByRole("option")).toHaveLength(2); + expect(screen.getByRole("option", { name: "custom-chat", exact: true })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "custom-judge", exact: true })).toBeInTheDocument(); + await user.click(screen.getByRole("option", { name: "custom-judge", exact: true })); + await user.click(screen.getByPlaceholderText("Select a baseline model")); + expect(screen.getAllByRole("option")).toHaveLength(2); + expect(screen.getByRole("option", { name: "custom-chat", exact: true })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "custom-judge", exact: true })).toBeInTheDocument(); + await user.click(screen.getByRole("option", { name: "custom-chat", exact: true })); + await user.click(screen.getByText("Start shadow eval")); + expect(start.mutate).toHaveBeenCalledWith( + expect.objectContaining({ judge_model: "custom-judge", baseline_model: "custom-chat", models: [] }), + ); + }); + }); + client.clear(); + }); + it("offers the start form while the list is still loading", () => { mockHooks({ isPending: true }); render(); @@ -444,7 +532,8 @@ describe("ShadowEvalSection", () => { expect(screen.getByText("Start shadow eval")).toBeDisabled(); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + expect(screen.queryByRole("option", { name: /openai\/gpt-4o/ })).not.toBeInTheDocument(); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByText("Start shadow eval")); const expectedBody = { @@ -457,7 +546,7 @@ describe("ShadowEvalSection", () => { shadow_percentage: 10, duration_days: 7, max_budget: 10, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); @@ -474,7 +563,7 @@ describe("ShadowEvalSection", () => { await user.click(within(teamList).getByText("engineering")); await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByText("Start shadow eval")); const expectedBody = { @@ -487,7 +576,7 @@ describe("ShadowEvalSection", () => { shadow_percentage: 10, duration_days: 7, max_budget: 10, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); @@ -503,7 +592,7 @@ describe("ShadowEvalSection", () => { await chooseSelectOption(user, screen.getByPlaceholderText("Every model the targets use"), "prod-claude"); await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByText("Start shadow eval")); expect(start.mutate).toHaveBeenCalledWith( @@ -524,20 +613,23 @@ describe("ShadowEvalSection", () => { expect(screen.queryByPlaceholderText("Select a baseline model")).not.toBeInTheDocument(); expect(screen.getByPlaceholderText("Every model the targets use")).toBeInTheDocument(); - await user.click(screen.getByText("Adoption check: key's traffic vs the router")); - await user.click(await screen.findByText("Regression check: router's picks vs a baseline")); + await chooseSelectOption( + user, + screen.getByText("Adoption check: key's traffic vs the router"), + "Regression check: router's picks vs a baseline", + ); expect(screen.queryByPlaceholderText("Every model the targets use")).not.toBeInTheDocument(); await user.click(screen.getByPlaceholderText("Search keys by alias")); const keyList = await screen.findByTestId("paginated-multi-select-list"); await user.click(within(keyList).getByText("prod-alpha")); await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); expect(screen.getByText("Start shadow eval")).toBeDisabled(); await user.click(screen.getByPlaceholderText("Select a baseline model")); - expect(await screen.findByRole("option", { name: /openai\/gpt-4o/ })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: /openai\/gpt-4o/ })).not.toBeInTheDocument(); await user.click(screen.getByRole("option", { name: /prod-claude/ })); await user.click(screen.getByText("Start shadow eval")); @@ -552,7 +644,7 @@ describe("ShadowEvalSection", () => { shadow_percentage: 10, duration_days: 7, max_budget: 10, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); @@ -574,7 +666,7 @@ describe("ShadowEvalSection", () => { screen.getByText("Every router sees the same sampled requests, judged against the same live responses"), ).toBeInTheDocument(); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByText("Start shadow eval")); const expectedBody = { @@ -587,7 +679,7 @@ describe("ShadowEvalSection", () => { shadow_percentage: 10, duration_days: 7, max_budget: 10, - judge_model: "anthropic/claude-sonnet-5", + judge_model: "prod-judge", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); @@ -605,10 +697,13 @@ describe("ShadowEvalSection", () => { await user.click(await screen.findByText("gpt-auto")); await user.click(routerInput); await user.click(await screen.findByText("claude-auto")); - await user.click(screen.getByText("Adoption check: key's traffic vs the router")); - await user.click(await screen.findByText("Regression check: router's picks vs a baseline")); + await chooseSelectOption( + user, + screen.getByText("Adoption check: key's traffic vs the router"), + "Regression check: router's picks vs a baseline", + ); await user.click(screen.getByPlaceholderText("Select a judge model")); - await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(await screen.findByRole("option", { name: /prod-judge/ })); await user.click(screen.getByPlaceholderText("Select a baseline model")); await user.click(screen.getByRole("option", { name: /prod-claude/ })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx index 2eb5fa9c945..d85d26a21a8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx @@ -5,8 +5,13 @@ import React, { useMemo, useState } from "react"; import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; -import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; +import { + useAutoRouters, + usePlainChatModelDeployments, + usePlainChatModelGroups, + usePlainModelGroups, +} from "@/app/(dashboard)/hooks/models/useModels"; +import { buildModelAvailability, deploymentRefsFromModelInfo, resolveAvailableModels } from "@/lib/autorouter_presets"; import { MultiSelect } from "@/components/shared/MultiSelect"; import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect"; import TeamMultiSelect from "@/components/common_components/team_multi_select"; @@ -24,53 +29,8 @@ type ShadowEvalDirection = ShadowEvalJob["direction"]; const MAX_ROUTERS = 4; const MAX_MODELS = 100; - const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const; -interface CostMapEntry { - litellm_provider?: string; - mode?: string; -} - -const useChatModelNames = (): string[] => { - const { data: costMap } = useModelCostMap(); - return useMemo(() => { - if (!costMap) return []; - const chatModels = Object.entries(costMap as Record) - .filter(([, value]) => value?.mode === "chat" && value?.litellm_provider) - .map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`)); - return [...new Set(chatModels)].toSorted((a, b) => a.localeCompare(b)); - }, [costMap]); -}; - -const useJudgeModelOptions = (): SearchSelectOption[] => { - const chatModels = useChatModelNames(); - return useMemo(() => { - const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({ - label: model, - value: model, - sublabel: "Recommended", - })); - const pinnedNames = new Set(RECOMMENDED_JUDGE_MODELS); - const rest = chatModels.filter((model) => !pinnedNames.has(model)).map((model) => ({ label: model, value: model })); - return [...pinned, ...rest]; - }, [chatModels]); -}; - -const useBaselineModelOptions = (): SearchSelectOption[] => { - const configuredGroups = usePlainModelGroups(); - const chatModels = useChatModelNames(); - return useMemo(() => { - const configured = [...configuredGroups] - .toSorted((a, b) => a.localeCompare(b)) - .map((model) => ({ label: model, value: model, sublabel: "Configured on this gateway" })); - const rest = chatModels - .filter((model) => !configuredGroups.has(model)) - .map((model) => ({ label: model, value: model })); - return [...configured, ...rest]; - }, [configuredGroups, chatModels]); -}; - const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[] = [ { value: "forward", label: "Adoption check: key's traffic vs the router" }, { value: "reverse", label: "Regression check: router's picks vs a baseline" }, @@ -276,13 +236,32 @@ export const StartForm: React.FC = () => { const [judgeModel, setJudgeModel] = useState(""); const [maxBudget, setMaxBudget] = useState("10"); const { data: autoRouters } = useAutoRouters(); - const judgeModelOptions = useJudgeModelOptions(); - const baselineModelOptions = useBaselineModelOptions(); const configuredGroups = usePlainModelGroups(); + const chatGroups = usePlainChatModelGroups(); + const chatDeployments = usePlainChatModelDeployments(); const modelOptions = useMemo( () => [...configuredGroups].toSorted((a, b) => a.localeCompare(b)).map((name) => ({ label: name, value: name })), [configuredGroups], ); + const chatOptions = useMemo( + () => modelOptions.filter((option) => chatGroups.has(option.value)), + [modelOptions, chatGroups], + ); + const chatAvailability = useMemo( + () => buildModelAvailability(chatGroups, deploymentRefsFromModelInfo(chatDeployments)), + [chatDeployments, chatGroups], + ); + const recommendedJudgeModels = useMemo( + () => new Set(RECOMMENDED_JUDGE_MODELS.flatMap((model) => resolveAvailableModels(model, chatAvailability))), + [chatAvailability], + ); + const judgeOptions = useMemo( + () => + chatOptions.map((option) => + recommendedJudgeModels.has(option.value) ? { ...option, sublabel: "Recommended" } : option, + ), + [chatOptions, recommendedJudgeModels], + ); const start = useStartShadowEval(); const routerOptions = useMemo(() => { @@ -434,7 +413,7 @@ export const StartForm: React.FC = () => { {direction === "reverse" && ( { )} { }); }); -describe("selectPlainModelGroups", () => { - it("keeps only non-auto-router model groups", () => { - const deployments: AutoRouterCandidateDeployment[] = [ - { model_name: "smart-router", litellm_params: { model: "auto_router/complexity_router" } }, - { model_name: "claude-haiku", litellm_params: { model: "anthropic/claude-haiku-4-5" } }, - { model_name: "claude-sonnet", litellm_params: { model: "anthropic/claude-sonnet-4-5" } }, - { model_name: "cheap-router", litellm_params: { model: "auto_router/adaptive_router" } }, +describe("selectPlainChatModelGroups", () => { + it("keeps chat-capable groups when mode metadata is absent or any sibling is compatible", () => { + const deployments: AutoRouterDeployment[] = [ + { model_name: "no-info" }, + { model_name: "null-info", model_info: null }, + { model_name: "empty-info", model_info: {} }, + { model_name: "missing-mode", model_info: { db_model: false } }, + { model_name: "null-mode", model_info: { mode: null } }, + { model_name: "empty-mode", model_info: { mode: "" } }, + { model_name: "chat", model_info: { mode: "chat", db_model: true } }, + { model_name: "completion", model_info: { mode: "completion" } }, + { model_name: "chat-and-missing", model_info: { mode: "chat" } }, + { model_name: "chat-and-missing" }, + { model_name: "chat-then-embedding", model_info: { mode: "chat" } }, + { model_name: "chat-then-embedding", model_info: { mode: "embedding" } }, + { model_name: "embedding-then-chat", model_info: { mode: "embedding" } }, + { model_name: "embedding-then-chat", model_info: { mode: "chat" } }, + { model_name: "embedding-only", model_info: { mode: "embedding" } }, + { model_name: "speech-only", model_info: { mode: "speech" } }, + { model_name: "shared-router", litellm_params: { model: "openai/gpt-4o" } }, + { model_name: "shared-router", litellm_params: { model: "auto_router/complexity_router" } }, + { model_name: "", model_info: { mode: "chat" } }, ]; - expect(selectPlainModelGroups(deployments)).toEqual(new Set(["claude-haiku", "claude-sonnet"])); - }); - - it("drops a group name that also fronts an auto-router deployment", () => { - const deployments: AutoRouterCandidateDeployment[] = [ - { model_name: "shared-name", litellm_params: { model: "auto_router/complexity_router" } }, - { model_name: "shared-name", litellm_params: { model: "anthropic/claude-sonnet-4-5" } }, - ]; - - expect(selectPlainModelGroups(deployments)).toEqual(new Set()); - }); - - it("drops deployments that have no public model_name", () => { - expect(selectPlainModelGroups([{ model_name: "", litellm_params: { model: "openai/gpt-4o" } }])).toEqual(new Set()); + expect(selectPlainChatModelGroups(deployments)).toEqual( + new Set([ + "no-info", + "null-info", + "empty-info", + "missing-mode", + "null-mode", + "empty-mode", + "chat", + "completion", + "chat-and-missing", + "chat-then-embedding", + "embedding-then-chat", + ]), + ); }); }); @@ -1103,6 +1121,47 @@ describe("useAutoRouterModelGroups", () => { expect(modelInfoCall).toHaveBeenCalledWith("test-access-token", "test-user-id", "Admin", 3, 1000); }); + it("uses every page for configured chat groups and keeps custom deployments without mode metadata", async () => { + (modelInfoCall as any).mockImplementation((_t: string, _u: string, _r: string, page: number) => + Promise.resolve( + page === 1 + ? { + data: [ + { model_name: "configured-chat", model_info: { mode: "chat" } }, + { model_name: "embedding-only", model_info: { mode: "embedding" } }, + ], + total_pages: 2, + } + : { + data: [ + { model_name: "custom-no-mode", model_info: { db_model: true } }, + { model_name: "speech-only", model_info: { mode: "speech" } }, + ], + total_pages: 2, + }, + ), + ); + + const { result } = renderHook(() => usePlainChatModelGroups(), { wrapper }); + + await waitFor(() => expect(result.current.size).toBe(2)); + expect(result.current).toEqual(new Set(["configured-chat", "custom-no-mode"])); + expect(modelInfoCall).toHaveBeenCalledTimes(2); + }); + + it("returns an empty chat group set while loading and after failure", async () => { + (modelInfoCall as any).mockReturnValueOnce(new Promise(() => {})); + const loading = renderHook(() => usePlainChatModelGroups(), { wrapper }); + expect(loading.result.current).toEqual(new Set()); + loading.unmount(); + + queryClient.clear(); + (modelInfoCall as any).mockRejectedValueOnce(new Error("boom")); + const failed = renderHook(() => usePlainChatModelGroups(), { wrapper }); + await waitFor(() => expect(modelInfoCall).toHaveBeenCalledTimes(2)); + expect(failed.result.current).toEqual(new Set()); + }); + it("returns an empty set before the model list resolves", () => { (modelInfoCall as any).mockReturnValue(new Promise(() => {})); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index b3a783a71dc..579ee7ff81a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -2,6 +2,7 @@ import { useQuery, useInfiniteQuery, useQueryClient, UseQueryResult } from "@tan import { createQueryKeys } from "../common/queryKeysFactory"; import { modelInfoCall, modelHubCall, modelAvailableCall } from "@/components/networking"; import useAuthorized from "../useAuthorized"; +import { EndpointType, isModeCompatibleWithEndpoint } from "@/components/chat_ui/mode_endpoint_mapping"; export interface ProxyModel { id: string; @@ -87,6 +88,7 @@ export const useModelsInfo = ( const AUTO_ROUTER_MODEL_PREFIX = "auto_router/"; const AUTO_ROUTER_LOOKUP_PAGE_SIZE = 1000; const NO_AUTO_ROUTERS: ReadonlySet = new Set(); +const NO_DEPLOYMENTS: AutoRouterDeployment[] = []; export interface AutoRouterCandidateDeployment { model_name?: string | null; @@ -96,6 +98,7 @@ export interface AutoRouterCandidateDeployment { export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { litellm_params?: { model?: string | null; + base_model?: string | null; complexity_router_config?: unknown; complexity_router_default_model?: string | null; auto_router_config?: unknown; @@ -111,6 +114,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { /** False for config.yaml-defined deployments, which the update and delete routes refuse. */ db_model?: boolean | null; base_model?: string | null; + mode?: string | null; created_at?: string | null; updated_at?: string | null; team_id?: string | null; @@ -142,6 +146,22 @@ export const selectPlainModelGroups = (deployments: AutoRouterCandidateDeploymen ); }; +export const selectPlainChatModelDeployments = (deployments: AutoRouterDeployment[]): AutoRouterDeployment[] => { + const plainGroups = selectPlainModelGroups(deployments); + return deployments.filter( + (deployment) => + plainGroups.has(deployment.model_name ?? "") && + isModeCompatibleWithEndpoint(deployment.model_info?.mode, EndpointType.CHAT), + ); +}; + +export const selectPlainChatModelGroups = (deployments: AutoRouterDeployment[]): ReadonlySet => + new Set( + selectPlainChatModelDeployments(deployments) + .map((deployment) => deployment.model_name) + .filter((name): name is string => Boolean(name)), + ); + export const fetchAllModelDeployments = async ( accessToken: string, userId: string, @@ -180,37 +200,32 @@ export const autoRouterListKey = (userId: string | null, userRole: string | null }, }); -export const useAutoRouterModelGroups = (): ReadonlySet => { +const useDeployments = ( + select: (deployments: AutoRouterDeployment[]) => TSelected, +): UseQueryResult => { const { accessToken, userId, userRole } = useAuthorized(); - const { data } = useQuery>({ + return useQuery({ queryKey: autoRouterListKey(userId, userRole), queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!), enabled: Boolean(accessToken && userId && userRole), - select: selectAutoRouterModelGroups, + select, }); - return data ?? NO_AUTO_ROUTERS; }; -export const usePlainModelGroups = (): ReadonlySet => { - const { accessToken, userId, userRole } = useAuthorized(); - const { data } = useQuery>({ - queryKey: autoRouterListKey(userId, userRole), - queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!), - enabled: Boolean(accessToken && userId && userRole), - select: selectPlainModelGroups, - }); - return data ?? NO_AUTO_ROUTERS; -}; +export const useAutoRouterModelGroups = (): ReadonlySet => + useDeployments(selectAutoRouterModelGroups).data ?? NO_AUTO_ROUTERS; -export const useAutoRouters = (): UseQueryResult => { - const { accessToken, userId, userRole } = useAuthorized(); - return useQuery({ - queryKey: autoRouterListKey(userId, userRole), - queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!), - enabled: Boolean(accessToken && userId && userRole), - select: selectAutoRouterDeployments, - }); -}; +export const usePlainModelGroups = (): ReadonlySet => + useDeployments(selectPlainModelGroups).data ?? NO_AUTO_ROUTERS; + +export const usePlainChatModelGroups = (): ReadonlySet => + useDeployments(selectPlainChatModelGroups).data ?? NO_AUTO_ROUTERS; + +export const usePlainChatModelDeployments = (): AutoRouterDeployment[] => + useDeployments(selectPlainChatModelDeployments).data ?? NO_DEPLOYMENTS; + +export const useAutoRouters = (): UseQueryResult => + useDeployments(selectAutoRouterDeployments); export const useInvalidateAutoRouters = (): (() => Promise) => { const queryClient = useQueryClient(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointUtils.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointUtils.tsx index 8fd4a57dbfd..87d569f6490 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointUtils.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointUtils.tsx @@ -1,7 +1,9 @@ import { ModelGroup } from "@/components/llm_calls/fetch_models"; -import { EndpointType, getEndpointType, ModelMode } from "@/components/chat_ui/mode_endpoint_mapping"; - -const KNOWN_MODEL_MODES = new Set(Object.values(ModelMode)); +import { + EndpointType, + getEndpointType, + isModeCompatibleWithEndpoint, +} from "@/components/chat_ui/mode_endpoint_mapping"; export const determineEndpointType = (selectedModel: string, modelInfo: ModelGroup[]): EndpointType => { const selectedModelInfo = modelInfo.find((option) => option.model_group === selectedModel); @@ -13,31 +15,8 @@ export const determineEndpointType = (selectedModel: string, modelInfo: ModelGro return EndpointType.CHAT; }; -export const isModelCompatibleWithEndpoint = (model: ModelGroup, endpointType: EndpointType): boolean => { - if (!model.mode) { - return true; - } - - if (!KNOWN_MODEL_MODES.has(model.mode)) { - return false; - } - - const optionEndpoint = getEndpointType(model.mode); - - if ( - endpointType === EndpointType.RESPONSES || - endpointType === EndpointType.ANTHROPIC_MESSAGES || - endpointType === EndpointType.INTERACTIONS - ) { - return optionEndpoint === endpointType || optionEndpoint === EndpointType.CHAT; - } - - if (endpointType === EndpointType.IMAGE_EDITS) { - return optionEndpoint === endpointType || optionEndpoint === EndpointType.IMAGE; - } - - return optionEndpoint === endpointType; -}; +export const isModelCompatibleWithEndpoint = (model: ModelGroup, endpointType: EndpointType): boolean => + isModeCompatibleWithEndpoint(model.mode, endpointType); export const filterModelsForEndpoint = (models: ModelGroup[], endpointType: EndpointType): ModelGroup[] => models.filter((model) => isModelCompatibleWithEndpoint(model, endpointType)); diff --git a/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx b/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx index 930ded5d1a5..b46200ccbf2 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx @@ -57,3 +57,20 @@ export const getEndpointType = (mode: string): EndpointType => { // else default to chat return EndpointType.CHAT; }; + +export const isModeCompatibleWithEndpoint = (mode: string | null | undefined, endpointType: EndpointType): boolean => { + if (!mode) return true; + if (!Object.values(ModelMode).includes(mode as ModelMode)) return false; + const optionEndpoint = getEndpointType(mode); + if ( + endpointType === EndpointType.RESPONSES || + endpointType === EndpointType.ANTHROPIC_MESSAGES || + endpointType === EndpointType.INTERACTIONS + ) { + return optionEndpoint === endpointType || optionEndpoint === EndpointType.CHAT; + } + if (endpointType === EndpointType.IMAGE_EDITS) { + return optionEndpoint === endpointType || optionEndpoint === EndpointType.IMAGE; + } + return optionEndpoint === endpointType; +}; diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 8a5f83adbdf..fed11454c23 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -13,6 +13,7 @@ import { buildModelAvailability, deploymentRefsFromModelInfo, normalizeModelName, + resolveAvailableModels, } from "./autorouter_presets"; import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching"; import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords"; @@ -380,6 +381,18 @@ describe("autorouter_presets", () => { expect(availability.underlyingIndex.size).toBe(0); }); + it("returns every configured group serving the same underlying model", () => { + const availability = buildModelAvailability( + ["z-group", "a-group"], + [ + { modelGroup: "z-group", underlyingModels: ["anthropic/claude-sonnet-5"] }, + { modelGroup: "a-group", underlyingModels: ["bedrock/us.anthropic.claude-sonnet-5-v1:0"] }, + ], + ); + + expect(resolveAvailableModels("anthropic/claude-sonnet-5", availability)).toEqual(["a-group", "z-group"]); + }); + it("breaks ties between groups serving the same model deterministically, alphabetically", () => { const availability = buildModelAvailability( ["z-group", "a-group"], diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index f2df55fc310..02096cada41 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -159,16 +159,19 @@ export const deploymentRefsFromModelInfo = ( return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : []; }); -export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => { +export const resolveAvailableModels = (requiredModel: string, availability: ModelAvailability): readonly string[] => { const { modelGroups, underlyingIndex } = availability; - if (modelGroups.has(requiredModel)) return requiredModel; + if (modelGroups.has(requiredModel)) return [requiredModel]; const normalized = normalizeModelName(requiredModel); - const groupMatch = Array.from(modelGroups).find((available) => normalizeModelName(available) === normalized); - if (groupMatch !== undefined) return groupMatch; + const groupMatches = Array.from(modelGroups).filter((available) => normalizeModelName(available) === normalized); + if (groupMatches.length > 0) return groupMatches; const key = normalizeUnderlyingModel(requiredModel); - return key === null ? undefined : underlyingIndex.get(key)?.[0]; + return key === null ? [] : underlyingIndex.get(key) ?? []; }; +export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => + resolveAvailableModels(requiredModel, availability)[0]; + export const getMissingModels = ( config: Parameters[0], availability: ModelAvailability, From 493c98f9bffbcf2ea1d5ba30e324427daa9a173b Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:11:20 -0700 Subject: [PATCH 269/411] fix(mcp): normalize default ports in preview origins --- .../_experimental/mcp_server/rest_endpoints.py | 5 ++++- .../mcp_server/test_rest_endpoints.py | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index a116c09d844..0c919ac7657 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1261,8 +1261,11 @@ if MCP_AVAILABLE: else None ) saved_origin: Final = _redact_mcp_resource_url(saved_server.url) if saved_server else None + preview_origin: Final = _redact_mcp_resource_url(new_mcp_server_request.url) may_inherit: Final = new_mcp_server_request.auth_type not in _STAGED_AUTH_VALUE_AUTH_TYPES or ( - saved_origin is not None and saved_origin == _redact_mcp_resource_url(new_mcp_server_request.url) + saved_origin is not None + and preview_origin is not None + and httpx.URL(saved_origin) == httpx.URL(preview_origin) ) request: Final = ( _inherit_credentials_from_existing_server(new_mcp_server_request) if may_inherit else new_mcp_server_request diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index ee5dabf043f..9b0bf1d92f6 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -154,11 +154,17 @@ class TestExecuteWithMcpClient: assert saved.url == "https://stored.example/mcp" @pytest.mark.parametrize( - "url", ("https://other.example/mcp", "http://stored.example/mcp", "https://stored.example:8443/mcp") + ("url", "same_origin"), + ( + ("https://other.example/mcp", False), + ("http://stored.example/mcp", False), + ("https://stored.example:8443/mcp", False), + ("https://stored.example:443/mcp", True), + ), ) @pytest.mark.parametrize("explicit_credential", (None, "preview:explicit")) - def test_static_preview_does_not_inherit_credentials_across_origins( - self, monkeypatch: pytest.MonkeyPatch, url: str, explicit_credential: str | None + def test_static_preview_respects_origin_when_inheriting_credentials( + self, monkeypatch: pytest.MonkeyPatch, url: str, same_origin: bool, explicit_credential: str | None ) -> None: from starlette.datastructures import Headers @@ -185,8 +191,9 @@ class TestExecuteWithMcpClient: credentials={"auth_value": explicit_credential} if explicit_credential else None, ) staged: Final = rest_endpoints._stage_server_test(payload, Headers()) - assert staged.mcp_auth_header == explicit_credential - assert staged.request.credentials == payload.credentials + expected: Final = explicit_credential or ("preview:stored" if same_origin else None) + assert staged.mcp_auth_header == expected + assert staged.request.credentials == ({"auth_value": expected} if expected else None) @pytest.mark.asyncio async def test_redacts_stack_trace(self, monkeypatch): From 47eb7f434974604a488e5a11193b3ef43410259c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:17:32 -0700 Subject: [PATCH 270/411] fix(token_counter): give each event loop its own count limiter A single process-wide anyio.CapacityLimiter wakes waiters with the asyncio.Event of whichever loop created it, so a fifth loop in another thread (asyncio.run per request under Celery, gunicorn sync workers, or run_async_function from function_with_fallbacks) waited forever once four counts were in flight, and building it at import time raised AsyncLibraryNotFoundError on anyio below 4.2. The limiter now lives in a RunVar and is created on first use inside the running loop. TOKEN_COUNTER_MAX_CONCURRENT_COUNTS reads from the environment like TOKEN_COUNTER_MAX_EXACT_CHARS, and the proxy's tokenizer lookup runs on the shared thread pool so a Hub download no longer holds a count slot. --- litellm/constants.py | 7 +++- litellm/litellm_core_utils/token_counter.py | 20 +++++++++- litellm/proxy/proxy_server.py | 3 +- .../litellm_core_utils/test_token_counter.py | 38 +++++++++++++++++++ 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 0bd7def539d..d8c83876e87 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -397,7 +397,12 @@ TOKEN_COUNTER_MAX_EXACT_CHARS: Final = get_env_int_in_range( minimum=1, maximum=1_000_000_000, ) -TOKEN_COUNTER_MAX_CONCURRENT_COUNTS: Final = 4 +TOKEN_COUNTER_MAX_CONCURRENT_COUNTS: Final = get_env_int_in_range( + "TOKEN_COUNTER_MAX_CONCURRENT_COUNTS", + default=4, + minimum=1, + maximum=256, +) MAX_TILE_WIDTH: Final = int(os.getenv("MAX_TILE_WIDTH", 512)) MAX_TILE_HEIGHT: Final = int(os.getenv("MAX_TILE_HEIGHT", 512)) OPENAI_FILE_SEARCH_COST_PER_1K_CALLS: Final = float(os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000)) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index e71ce5dea9c..70562748c6a 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -7,6 +7,7 @@ from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from typing import Final, Literal, cast import anyio +import anyio.lowlevel import httpx import tiktoken from tokenizers import Tokenizer @@ -326,13 +327,28 @@ Type for a function that counts tokens in a string. EXTRAPOLATION_SAMPLES: Final = 16 T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") -_COUNT_OFFLOAD_LIMITER: Final = anyio.CapacityLimiter(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS) +_COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") + + +def _count_offload_limiter_for_this_loop() -> anyio.CapacityLimiter: + existing: Final = _COUNT_OFFLOAD_LIMITER.get(None) + if existing is not None: + return existing + created: Final = anyio.CapacityLimiter(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS) + _COUNT_OFFLOAD_LIMITER.set(created) + return created def offload_token_count( function: Callable[T_ParamSpec, T_Retval], ) -> Callable[T_ParamSpec, Awaitable[T_Retval]]: - return asyncify(function, limiter=_COUNT_OFFLOAD_LIMITER) + async def offloaded( + *args: T_ParamSpec.args, + **kwargs: T_ParamSpec.kwargs, # kwargs-ok: ParamSpec keeps the wrapped function's own keyword contract + ) -> T_Retval: + return await asyncify(function, limiter=_count_offload_limiter_for_this_loop())(*args, **kwargs) + + return offloaded def _get_tiktoken_count_function( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0e3e5c251c8..453f57131b5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -62,6 +62,7 @@ from litellm.constants import ( LITELLM_UI_SESSION_DURATION, RUNTIME_UPDATABLE_ROUTER_SETTINGS, ) +from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, ) @@ -12713,7 +12714,7 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False) CustomHuggingfaceTokenizer | None, model_info.get("custom_tokenizer", None), ) - _tokenizer_used: Final = await offload_token_count(litellm.utils._select_tokenizer)( + _tokenizer_used: Final = await asyncify(litellm.utils._select_tokenizer)( model=model_to_use, custom_tokenizer=custom_tokenizer ) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 7cc7c6c763c..7da04d12569 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -5,6 +5,7 @@ import importlib import threading import time import traceback +from concurrent.futures import Future, wait from typing import Final from unittest.mock import MagicMock @@ -203,6 +204,43 @@ async def test_offloaded_counts_do_not_borrow_from_the_shared_thread_pool(): assert 1 < encoder.peak_in_flight <= TOKEN_COUNTER_MAX_CONCURRENT_COUNTS +def _count_in_a_fresh_event_loop(text: str, result: Future[int]) -> None: + def slow_count(counted: str) -> int: + time.sleep(0.1) + return len(counted) + + result.set_result(asyncio.run(offload_token_count(slow_count)(text))) + + +def test_offloaded_counts_finish_in_every_event_loop_that_shares_the_process(): + loops: Final = 2 * TOKEN_COUNTER_MAX_CONCURRENT_COUNTS + results: Final = tuple(Future[int]() for _ in range(loops)) + threads: Final = tuple( + threading.Thread(target=_count_in_a_fresh_event_loop, args=("a" * size, result), daemon=True) + for size, result in enumerate(results, start=1) + ) + for thread in threads: + thread.start() + + _, pending = wait(results, timeout=5) + + assert not pending + assert tuple(result.result() for result in results) == tuple(range(1, loops + 1)) + + +@pytest.mark.parametrize( + ("configured", "expected"), + [("8", 8), ("0", 4), ("not-an-int", 4)], +) +def test_max_concurrent_counts_config_is_honoured(monkeypatch: pytest.MonkeyPatch, configured: str, expected: int): + monkeypatch.setenv("TOKEN_COUNTER_MAX_CONCURRENT_COUNTS", configured) + try: + assert importlib.reload(litellm.constants).TOKEN_COUNTER_MAX_CONCURRENT_COUNTS == expected + finally: + monkeypatch.delenv("TOKEN_COUNTER_MAX_CONCURRENT_COUNTS") + importlib.reload(litellm.constants) + + def test_token_counter_applies_the_default_cap(): max_exact_chars: Final = litellm.constants.TOKEN_COUNTER_MAX_EXACT_CHARS prose: Final = ("The quick brown fox jumps over the lazy dog. " * (max_exact_chars // 45 + 1))[:max_exact_chars] From 3aa9267df278a4a28399460cc7cbe4708a4bd169 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 9 Sep 2026 18:19:52 -0700 Subject: [PATCH 271/411] fix(ui): drop blank streamed costs instead of reading them as zero Number("") and Number(" ") both return 0, which passed the finite check, so a provider reporting an empty cost got a fabricated $0.000000 metric instead of having the unusable value omitted. Both ingestion sites carried the same inline parsing, so this pulls it into one parseUsageCost helper that keeps finite numbers and non-blank numeric strings and drops everything else, including booleans, arrays and breakdown objects. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YNw8WvkvCSeTcE5qvergu3 --- .../llm_calls/chat_completion.test.tsx | 6 +++ .../components/llm_calls/chat_completion.tsx | 10 ++--- .../llm_calls/responses_api.test.tsx | 29 ++++++++++++++ .../components/llm_calls/responses_api.tsx | 9 ++--- .../components/llm_calls/usage_cost.test.ts | 39 +++++++++++++++++++ .../src/components/llm_calls/usage_cost.ts | 23 +++++++++++ 6 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/llm_calls/usage_cost.test.ts create mode 100644 ui/litellm-dashboard/src/components/llm_calls/usage_cost.ts diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx index 129d51d50a2..71f36bdb6db 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx @@ -472,6 +472,12 @@ describe("chat_completion prompt cache usage", () => { expect(usageData).toEqual(expect.not.objectContaining({ cost: expect.anything() })); }); + + it("omits cost when the provider reports a blank value", async () => { + const usageData = await captureUsage({ cost: " " }); + + expect(usageData).toEqual(expect.not.objectContaining({ cost: expect.anything() })); + }); }); describe("chat_completion response cache", () => { diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx index c12a2c3cfb4..ffa2877fbd9 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx @@ -5,6 +5,7 @@ import { VectorStoreSearchResponse } from "../chat_ui/types"; import { getProxyBaseUrl } from "@/components/networking"; import { MCPServer, MCPToolset, type MCPEvent } from "@/components/mcp_tools/types"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; +import { parseUsageCost } from "./usage_cost"; const completionAsSingleChunk = (completion: ChatCompletion): ChatCompletionChunk => ({ @@ -243,12 +244,9 @@ export async function makeOpenAIChatCompletionRequest( usageData.reasoningTokens = chunkWithUsage.usage.completion_tokens_details.reasoning_tokens; } - // Extract cost from usage object if available - if (chunkWithUsage.usage.cost !== undefined && chunkWithUsage.usage.cost !== null) { - const parsedCost = Number(chunkWithUsage.usage.cost); - if (Number.isFinite(parsedCost)) { - usageData.cost = parsedCost; - } + const parsedCost = parseUsageCost(chunkWithUsage.usage.cost); + if (parsedCost !== undefined) { + usageData.cost = parsedCost; } onUsageData(usageData); diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 7b13f0ac2e3..b55df89d5cc 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -260,6 +260,35 @@ describe("responses_api", () => { expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ cost: expect.anything() }), ""); }); + it("should omit cost when the proxy reports a blank cost", async () => { + async function* streamWithBlankCost() { + yield { + type: "response.completed", + response: { + id: "resp_blank_cost", + usage: { output_tokens: 12, input_tokens: 12, total_tokens: 24, cost: " " }, + }, + }; + } + mockResponsesCreate.mockResolvedValueOnce(streamWithBlankCost()); + + const onUsageData = vi.fn(); + + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, + undefined, + undefined, + undefined, + onUsageData, + ); + + expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ cost: expect.anything() }), ""); + }); + it("should replay MCP output items as events for a non-streaming response", async () => { mockResponsesCreate.mockReturnValueOnce( nonStreamingResponse({ diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index 63085a81b94..ce54c7c6b40 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -4,6 +4,7 @@ import { TokenUsage } from "../chat_ui/ResponseMetrics"; import { getProxyBaseUrl } from "@/components/networking"; import { toast } from "@/lib/toast"; import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; +import { parseUsageCost } from "./usage_cost"; import type { MCPEvent } from "@/components/mcp_tools/types"; import { MCPServer, MCPToolset } from "@/components/mcp_tools/types"; import { @@ -311,11 +312,9 @@ export async function makeOpenAIResponsesRequest( usageData.reasoningTokens = reasoningTokens; } - if (usage.cost !== undefined && usage.cost !== null) { - const parsedCost = Number(usage.cost); - if (Number.isFinite(parsedCost)) { - usageData.cost = parsedCost; - } + const parsedCost = parseUsageCost(usage.cost); + if (parsedCost !== undefined) { + usageData.cost = parsedCost; } onUsageData(usageData, mcpToolUsed); diff --git a/ui/litellm-dashboard/src/components/llm_calls/usage_cost.test.ts b/ui/litellm-dashboard/src/components/llm_calls/usage_cost.test.ts new file mode 100644 index 00000000000..ee1021c0629 --- /dev/null +++ b/ui/litellm-dashboard/src/components/llm_calls/usage_cost.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { parseUsageCost } from "./usage_cost"; + +describe("parseUsageCost", () => { + it("keeps finite numbers, including zero", () => { + expect(parseUsageCost(0)).toBe(0); + expect(parseUsageCost(0.000063)).toBe(0.000063); + }); + + it("keeps numeric strings", () => { + expect(parseUsageCost("0.00019")).toBe(0.00019); + expect(parseUsageCost(" 0.00019 ")).toBe(0.00019); + }); + + it("drops blank strings instead of fabricating a zero cost", () => { + expect(parseUsageCost("")).toBeUndefined(); + expect(parseUsageCost(" ")).toBeUndefined(); + expect(parseUsageCost("\t\n")).toBeUndefined(); + }); + + it("drops strings with a numeric prefix instead of truncating them", () => { + expect(parseUsageCost("1oops")).toBeUndefined(); + expect(parseUsageCost("0.5 USD")).toBeUndefined(); + }); + + it("drops non-finite numbers", () => { + expect(parseUsageCost(Number.NaN)).toBeUndefined(); + expect(parseUsageCost(Number.POSITIVE_INFINITY)).toBeUndefined(); + }); + + it("drops values that are not numbers or strings", () => { + expect(parseUsageCost(null)).toBeUndefined(); + expect(parseUsageCost(undefined)).toBeUndefined(); + expect(parseUsageCost(true)).toBeUndefined(); + expect(parseUsageCost([])).toBeUndefined(); + expect(parseUsageCost(["0.5"])).toBeUndefined(); + expect(parseUsageCost({ total_cost: 0.5 })).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/llm_calls/usage_cost.ts b/ui/litellm-dashboard/src/components/llm_calls/usage_cost.ts new file mode 100644 index 00000000000..79f56dcb2df --- /dev/null +++ b/ui/litellm-dashboard/src/components/llm_calls/usage_cost.ts @@ -0,0 +1,23 @@ +/** + * Providers and upstream gateways report `usage.cost` unvalidated: it arrives as a number, a numeric + * string, an empty string, or something non-numeric. A cost that does not resolve to a finite number + * must be dropped rather than coerced, because `NaN` survives `JSON.stringify` as `null` and crashes + * the metrics row on the next load. + */ +export function parseUsageCost(rawCost: unknown): number | undefined { + if (typeof rawCost === "number") { + return Number.isFinite(rawCost) ? rawCost : undefined; + } + + if (typeof rawCost !== "string") { + return undefined; + } + + const trimmed = rawCost.trim(); + if (trimmed === "") { + return undefined; + } + + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : undefined; +} From b3f2de058be47e4613b8e6ad27039af3bb58c7a1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:19:56 -0700 Subject: [PATCH 272/411] fix(cli): only guard the default settings.json with the lite up and autoroute backups on login --config-claude --- litellm/proxy/client/cli/commands/auth.py | 4 +- .../client/cli/commands/claude_settings.py | 6 +++ .../proxy/client/cli/test_auth_commands.py | 37 ++++++++++++++++--- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 8e766d94732..a0bf9254bbc 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -41,9 +41,9 @@ from litellm.litellm_core_utils.cli_token_utils import ( ) from .claude_settings import ( - SETTINGS_FILE_OWNERS, ClaudeSettingsError, claude_settings_path, + settings_file_owners, write_claude_settings, ) from .pkce_login import ( @@ -782,7 +782,7 @@ def _configure_claude_code(base_url: str) -> None: """Point Claude Code at base_url by patching the settings.json it reads.""" settings_path: Final = claude_settings_path(os.environ) try: - write_claude_settings(base_url, settings_path, SETTINGS_FILE_OWNERS) + write_claude_settings(base_url, settings_path, settings_file_owners(settings_path)) except ClaudeSettingsError as e: raise click.ClickException(f"Logged in, but could not configure Claude Code: {e}") click.echo(f"\nConfigured Claude Code: {settings_path} now routes through {base_url.rstrip('/')}.") diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index 524edcb899a..7e96d2a3a08 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -64,6 +64,11 @@ def claude_settings_path(environ: Mapping[str, str]) -> Path: return Path(config_dir).expanduser() / "settings.json" +def settings_file_owners(settings_path: Path) -> tuple[SettingsFileOwner, ...]: + """The commands whose backups guard settings_path: `lite up` and `lite autoroute up` only ever manage the default file.""" + return SETTINGS_FILE_OWNERS if settings_path == CLAUDE_SETTINGS_PATH else () + + def load_json_or_empty(path: Path) -> dict[str, JsonValue]: try: content: Final = path.read_bytes() if path.exists() else b"" @@ -197,5 +202,6 @@ __all__ = ( "load_json_or_empty", "merge_claude_settings", "resolve_api_key_helper", + "settings_file_owners", "write_claude_settings", ) diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 225396d78f6..9c3c02a5e85 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -1398,9 +1398,10 @@ class TestLoginConfigClaude: def setup_method(self): self.runner = CliRunner() - def _run_login(self, tmp_path, args, base_url="https://test.example.com"): + def _run_login(self, tmp_path, args, base_url="https://test.example.com", *, config_dir_env=None): settings_path = tmp_path / "claude" / "settings.json" backup_path = tmp_path / "claude_settings_backup.json" + env = {"CLAUDE_CONFIG_DIR": str(settings_path.parent)} if config_dir_env is None else config_dir_env poll_response = Mock() poll_response.status_code = 200 poll_response.json.return_value = { @@ -1417,17 +1418,19 @@ class TestLoginConfigClaude: patch("litellm.proxy.client.cli.commands.auth.save_cli_token"), patch("litellm.proxy.client.cli.interface.show_commands"), patch( - "litellm.proxy.client.cli.commands.auth.SETTINGS_FILE_OWNERS", + "litellm.proxy.client.cli.commands.claude_settings.SETTINGS_FILE_OWNERS", (SettingsFileOwner(backup_path, "lite up", "lite down"),), ), + patch( + "litellm.proxy.client.cli.commands.claude_settings.CLAUDE_SETTINGS_PATH", + tmp_path / "default-home" / ".claude" / "settings.json", + ), patch( "litellm.proxy.client.cli.commands.claude_settings.shutil.which", return_value="/usr/local/bin/lite", ), ): - result = self.runner.invoke( - login, args, obj={"base_url": base_url}, env={"CLAUDE_CONFIG_DIR": str(settings_path.parent)} - ) + result = self.runner.invoke(login, args, obj={"base_url": base_url}, env=env) return result, settings_path, backup_path def test_default_login_does_not_touch_claude_settings(self, tmp_path): @@ -1460,6 +1463,30 @@ class TestLoginConfigClaude: assert written["theme"] == "dark" assert written["env"]["KEEP"] == "me" + def test_flag_refuses_while_lite_up_holds_the_default_settings_file(self, tmp_path): + default_settings_path = tmp_path / "default-home" / ".claude" / "settings.json" + (tmp_path / "claude_settings_backup.json").write_text("{}") + + result, _settings_path, _backup_path = self._run_login( + tmp_path, ["--config-claude"], config_dir_env={"CLAUDE_CONFIG_DIR": ""} + ) + + assert result.exit_code != 0 + assert "Login successful!" in result.output + assert "`lite up` is currently managing" in result.output + assert "Run `lite down` first" in result.output + assert not default_settings_path.exists() + + def test_flag_writes_an_alternate_config_dir_even_while_lite_up_holds_the_default_file(self, tmp_path): + (tmp_path / "claude_settings_backup.json").write_text("{}") + + result, settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + + assert result.exit_code == 0, result.output + written = json.loads(settings_path.read_text()) + assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://test.example.com auth print-token" + assert f"Configured Claude Code: {settings_path} now routes through https://test.example.com." in result.output + def test_settings_failure_is_reported_without_claiming_login_failed(self, tmp_path): settings_path = tmp_path / "claude" / "settings.json" settings_path.parent.mkdir(parents=True) From 3169c80252e6e98f6c68ff77869244dd9788b796 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:21:55 -0700 Subject: [PATCH 273/411] fix(mcp): preserve IPv6 hosts when comparing preview origins --- .../mcp_server/rest_endpoints.py | 19 ++++++++++---- .../mcp_server/test_rest_endpoints.py | 25 +++++++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 0c919ac7657..27cb632c843 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1243,6 +1243,17 @@ if MCP_AVAILABLE: mcp_auth_header: str | None oauth2_headers: dict[str, str] | None + def _preview_origin(url: str | None) -> tuple[str, str, int | None] | None: + if not url: + return None + try: + parsed: Final = httpx.URL(url) + except httpx.InvalidURL: + return None + if parsed.scheme not in ("http", "https") or not parsed.host: + return None + return parsed.scheme, parsed.host, parsed.port + def _stage_server_test(new_mcp_server_request: NewMCPServerRequest, headers: Headers) -> _StagedServerTest: """ Resolve the credentials a not-yet-saved server config carries for a preview call. @@ -1260,12 +1271,10 @@ if MCP_AVAILABLE: if new_mcp_server_request.server_id else None ) - saved_origin: Final = _redact_mcp_resource_url(saved_server.url) if saved_server else None - preview_origin: Final = _redact_mcp_resource_url(new_mcp_server_request.url) + saved_origin: Final = _preview_origin(saved_server.url) if saved_server else None + preview_origin: Final = _preview_origin(new_mcp_server_request.url) may_inherit: Final = new_mcp_server_request.auth_type not in _STAGED_AUTH_VALUE_AUTH_TYPES or ( - saved_origin is not None - and preview_origin is not None - and httpx.URL(saved_origin) == httpx.URL(preview_origin) + saved_origin is not None and saved_origin == preview_origin ) request: Final = ( _inherit_credentials_from_existing_server(new_mcp_server_request) if may_inherit else new_mcp_server_request diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 9b0bf1d92f6..16c6aa128d0 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -154,17 +154,28 @@ class TestExecuteWithMcpClient: assert saved.url == "https://stored.example/mcp" @pytest.mark.parametrize( - ("url", "same_origin"), + ("saved_url", "url", "same_origin"), ( - ("https://other.example/mcp", False), - ("http://stored.example/mcp", False), - ("https://stored.example:8443/mcp", False), - ("https://stored.example:443/mcp", True), + ("https://stored.example/mcp", "https://other.example/mcp", False), + ("https://stored.example/mcp", "http://stored.example/mcp", False), + ("https://stored.example/mcp", "https://stored.example:8443/mcp", False), + ("https://stored.example/mcp", "https://stored.example:443/mcp", True), + ("http://stored.example/mcp", "http://stored.example:80/edited", True), + ("https://stored.example/mcp", "HTTPS://STORED.EXAMPLE/edited", True), + ("https://[::1]/mcp", "https://[::1]/edited", True), + ("https://[::1]/mcp", "https://[::1]:443/edited", True), + ("https://[::1]/mcp", "https://[::2]/edited", False), + ("https://stored.example/mcp", "https://stored.example:invalid/mcp", False), ), ) @pytest.mark.parametrize("explicit_credential", (None, "preview:explicit")) def test_static_preview_respects_origin_when_inheriting_credentials( - self, monkeypatch: pytest.MonkeyPatch, url: str, same_origin: bool, explicit_credential: str | None + self, + monkeypatch: pytest.MonkeyPatch, + saved_url: str, + url: str, + same_origin: bool, + explicit_credential: str | None, ) -> None: from starlette.datastructures import Headers @@ -174,7 +185,7 @@ class TestExecuteWithMcpClient: saved: Final = MCPServer( server_id="saved-preview-server", name="saved", - url="https://stored.example/mcp", + url=saved_url, transport=MCPTransport.http, auth_type=MCPAuth.basic, authentication_token="preview:stored", From 4adf99557f901fd73525697fdba8c9b2c99d0cc9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:27:12 -0700 Subject: [PATCH 274/411] fix(responses): echo "auto" for a tool_choice the bridge cannot express instead of failing after the provider call --- .../transformation.py | 6 +++-- .../test_litellm_completion_responses.py | 26 +++++++++++++++++++ .../test_streaming_iterator_transformation.py | 20 ++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 6b9931b7df2..fca5b0d11cf 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -287,8 +287,10 @@ class LiteLLMCompletionResponsesConfig: return ToolChoiceCustomParam(type="custom", name=custom_name) case _, {"type": "function", "function": {"name": str(function_name)}}: return ToolChoiceFunctionParam(type="function", name=function_name) - case _, normalized: - return _RESPONSES_API_TOOL_CHOICE_ADAPTER.validate_python(normalized) + case _, "none" | "auto" | "required" as normalized: + return normalized + case _, _: + return "auto" @staticmethod def _should_drop_derived_web_search_options(model: str, custom_llm_provider: str | None) -> bool: diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 8f2937cb252..46249e50572 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1435,6 +1435,9 @@ class TestToolChoiceTransformation: ("required", "required"), ("none", "none"), (None, "auto"), + ("any", "auto"), + ("run_command", "auto"), + ({"name": "run_command"}, "auto"), ], ) def test_transform_tool_choice_for_responses_api_response( @@ -1478,6 +1481,29 @@ class TestToolChoiceTransformation: assert responses_api_response.tool_choice == {"type": "function", "name": "run_command"} + def test_non_streamed_response_with_unrecognized_tool_choice_echoes_auto(self) -> None: + chat_completion_response: Final = ModelResponse( + id="chatcmpl-unrecognized-tool-choice", + created=1748575031, + model="claude-haiku-4-5", + object="chat.completion", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=Message(role="assistant", content="/Users/dev"), + ) + ], + ) + + responses_api_response: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Run the command pwd.", + responses_api_request={"tool_choice": "any"}, + chat_completion_response=chat_completion_response, + ) + + assert responses_api_response.tool_choice == "auto" + class TestContentTypeTransformation: """Test content type transformation from Responses API to Chat Completion format""" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 70f0957890d..850ee7ba623 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -685,3 +685,23 @@ def test_streamed_named_tool_choice_is_echoed_in_responses_api_shape() -> None: {"type": "function", "name": "run_command"}, ] assert any(getattr(event, "type", None) == "response.output_item.done" for event in events) + + +def test_streamed_unrecognized_tool_choice_is_echoed_as_auto() -> None: + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-haiku-4-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper([_tool_call_chunk(finish_reason="tool_calls")]), + request_input="Run the command pwd.", + responses_api_request={ + "tools": [{"type": "function", "name": "run_command", "parameters": {"type": "object"}}], + "tool_choice": "any", + }, + custom_llm_provider="anthropic", + litellm_metadata={}, + ) + + response_events: Final = [ + event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES + ] + + assert [event.response.tool_choice for event in response_events] == ["auto", "auto", "auto"] From 4716b46c243b90d833ea08de8bebcc9a20c963d2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:31:08 -0700 Subject: [PATCH 275/411] fix(router): read the affinity pin from the messages argument and strip bridge reasoning on a cross-group route The encrypted_content_affinity check only read the Anthropic history from request_kwargs["messages"], so a caller that passes it through the callback's messages argument alone skipped the pin. Read the argument first and fall back to the kwargs. When the minting deployment is not a candidate of the routed group, the base already strips the Responses input's encrypted reasoning; do the same for the bridge-tagged thinking blocks in Anthropic messages so the routed deployment gets the readable thinking text instead of ciphertext it cannot decrypt. --- .../prompt_templates/common_utils.py | 43 ++++++++++- .../encrypted_content_affinity_check.py | 14 +++- ...ore_utils_prompt_templates_common_utils.py | 60 ++++++++++++++- .../test_encrypted_content_affinity_check.py | 73 +++++++++++++++---- 4 files changed, 166 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index ae9b7f6bc4b..7ca2a8aad6a 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,7 +6,7 @@ import io import json import mimetypes import re -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from itertools import groupby from os import PathLike from pathlib import Path @@ -1889,6 +1889,47 @@ def is_encrypted_reasoning_block(block: object) -> bool: return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping)) +def strip_encrypted_reasoning_from_messages(messages: object) -> None: + """Drop the encrypted reasoning a routed deployment cannot decrypt from Anthropic-shaped + history, keeping the readable thinking text. + + Mutates the content lists in place: the router's fallback snapshot shares these + message objects, so a rebound list would replay the stripped blocks on the fallback hop. + """ + if not isinstance(messages, list): + return + for content in _anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json + _strip_encrypted_reasoning_from_blocks(content) + + +def _anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: + return ( + cast(list[object], content) # cast-ok: narrowed by isinstance + for message in messages + if isinstance(message, Mapping) + for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance + if isinstance(content, list) + ) + + +def _strip_encrypted_reasoning_from_blocks(content: object) -> None: + blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance + stripped: Final = tuple(_without_encrypted_reasoning_block(block) for block in blocks) + blocks[:] = (block for block in stripped if block is not None) # rebind-ok: list shared with fallback snapshot + + +def _without_encrypted_reasoning_block(block: object) -> object | None: + if not is_encrypted_reasoning_block(block): + return block + mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by is_encrypted_reasoning_block + if mapping.get("type") != "thinking" or not mapping.get("thinking"): + return None + kept: Final[dict[str, object]] = { # mutable-ok: thinking block rebuilt without the undecryptable signature + key: value for key, value in mapping.items() if key != "signature" + } + return kept + + def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: index, block = indexed_block return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary" diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index b2edb13a4bf..cdd70e6baf2 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -48,7 +48,10 @@ from litellm.exceptions import ( ServiceUnavailableError, ) from litellm.integrations.custom_logger import CustomLogger, Span -from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_content_of_block +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_content_of_block, + strip_encrypted_reasoning_from_messages, +) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.cooldown_cache import CooldownCacheValue from litellm.types.llms.openai import AllMessageValues @@ -274,8 +277,9 @@ class EncryptedContentAffinityCheck(CustomLogger): parent_otel_span: Span | None = None, ) -> list[dict]: """ - If the request ``input`` contains litellm-encoded item IDs, decode the - embedded ``model_id`` and pin the request to that deployment. Raises + If the request ``input`` contains litellm-encoded item IDs, or its Anthropic + ``messages`` replay a bridge-tagged thinking block, decode the embedded + ``model_id`` and pin the request to that deployment. Raises ``RateLimitError`` / ``ServiceUnavailableError`` when the originating deployment is a member of the routed model group but currently unavailable and no encryption-boundary peer exists, rather than dispatching a doomed @@ -304,9 +308,10 @@ class EncryptedContentAffinityCheck(CustomLogger): request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True request_input: Final = request_kwargs.get("input") + anthropic_messages: Final = messages or request_kwargs.get("messages") model_id: Final = self._extract_model_id_from_input( request_input - ) or self._extract_model_id_from_anthropic_messages(request_kwargs.get("messages")) + ) or self._extract_model_id_from_anthropic_messages(anthropic_messages) if not model_id: return typed_healthy_deployments @@ -363,6 +368,7 @@ class EncryptedContentAffinityCheck(CustomLogger): model, ) ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input) + strip_encrypted_reasoning_from_messages(anthropic_messages) return typed_healthy_deployments # The origin is a member of the routed group but currently unavailable (cooled down); fail fast diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index ecb07d8c937..71b71d55fa7 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1,3 +1,4 @@ +import copy import functools import json import os @@ -20,6 +21,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( is_encrypted_reasoning_block, responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, + strip_encrypted_reasoning_from_messages, update_messages_with_model_file_ids, ) @@ -1567,7 +1569,9 @@ class TestEncryptedReasoningReplay: def test_signature_round_trips_the_encrypted_content(self): assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes" - @pytest.mark.parametrize("signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7]) + @pytest.mark.parametrize( + "signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7] + ) def test_anything_else_is_not_encrypted_content(self, signature): assert encrypted_content_from_signature(signature) is None @@ -1576,7 +1580,11 @@ class TestEncryptedReasoningReplay: [{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}] ) assert items == ( - {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Plan."}], "encrypted_content": "gAAAA_1"}, + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Plan."}], + "encrypted_content": "gAAAA_1", + }, ) def test_encrypted_redacted_block_replays_with_an_empty_summary(self): @@ -1596,7 +1604,10 @@ class TestEncryptedReasoningReplay: ] ) assert items == ( - {"type": "reasoning", "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}]}, + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}], + }, {"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"}, {"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]}, ) @@ -1621,3 +1632,46 @@ class TestEncryptedReasoningReplay: ) def test_is_encrypted_reasoning_block(self, block, expected): assert is_encrypted_reasoning_block(block) is expected + + def test_strip_keeps_the_readable_thinking_and_drops_the_undecryptable_bytes(self): + assistant_content = [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("g2")}, + {"type": "thinking", "thinking": "", "signature": encrypted_reasoning_signature("g3")}, + {"type": "text", "text": "answer"}, + ] + messages = [ + {"role": "user", "content": "question"}, + {"role": "assistant", "content": assistant_content}, + {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, + ] + + strip_encrypted_reasoning_from_messages(messages) + + assert messages[1]["content"] is assistant_content + assert assistant_content == [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "thinking", "thinking": "packed by the bridge"}, + {"type": "text", "text": "answer"}, + ] + assert messages[0] == {"role": "user", "content": "question"} + assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]} + + @pytest.mark.parametrize( + "messages", + [ + "not a list", + None, + [{"role": "user", "content": None}], + [{"role": "user", "content": "plain string"}], + ["not a message"], + [{"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}]}], + ], + ) + def test_strip_leaves_history_without_bridge_reasoning_untouched(self, messages): + before = copy.deepcopy(messages) + + strip_encrypted_reasoning_from_messages(messages) + + assert messages == before diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 99a650436b8..57c02e43fc1 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1597,26 +1597,12 @@ async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_throu {"model_info": {"id": "openai-org-a"}, "litellm_params": {"model": "openai/gpt-5.1"}}, {"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5.1"}}, ] - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", "openai-org-b") - request_kwargs = { - "messages": [ - {"role": "user", "content": "Solve the zebra puzzle"}, - { - "role": "assistant", - "content": [ - {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, - {"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"}, - {"type": "text", "text": "The zebra owner lives in the green house."}, - ], - }, - {"role": "user", "content": "And who drinks water?"}, - ], - } + request_kwargs = {"model": "gpt-5.1"} pinned = await check.async_filter_deployments( model="gpt-5.1", healthy_deployments=deployments, - messages=None, + messages=_bridge_replayed_anthropic_messages(minted_by="openai-org-b"), request_kwargs=request_kwargs, ) @@ -1624,6 +1610,61 @@ async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_throu assert request_kwargs["_encrypted_content_affinity_pinned"] is True +def _bridge_replayed_anthropic_messages(minted_by: str) -> list: + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", minted_by) + return [ + {"role": "user", "content": "Solve the zebra puzzle"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, + {"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"}, + { + "type": "thinking", + "thinking": "The bridge packed this one", + "signature": f"litellm_encrypted_reasoning:{wrapped}", + }, + {"type": "text", "text": "The zebra owner lives in the green house."}, + ], + }, + {"role": "user", "content": "And who drinks water?"}, + ] + + +@pytest.mark.asyncio +async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_routed_to_another_group(): + """ + The /v1/messages twin of the tier-change case: the routed group holds no deployment + of the org that minted the reasoning, so the bridge-tagged blocks are stripped down + to their readable thinking text and the request dispatches to the routed pool. + """ + originating = _make_originating_mock(None, "key-a", model_name="gpt-reasoning-tier") + mock_router = _make_router_mock_with_cooldown( + originating, cooldown_entries=[], routed_group_model_ids=["openai-org-b"] + ) + check = EncryptedContentAffinityCheck(router=mock_router) + routed_pool = [{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5-nano"}}] + messages = _bridge_replayed_anthropic_messages(minted_by="openai-org-a") + assistant_content = messages[1]["content"] + request_kwargs = {"model": "gpt-5.1"} + + result = await check.async_filter_deployments( + model="gpt-simple-tier", + healthy_deployments=routed_pool, + messages=messages, + request_kwargs=request_kwargs, + ) + + assert result is routed_pool + assert "_encrypted_content_affinity_pinned" not in request_kwargs + assert messages[1]["content"] is assistant_content + assert assistant_content == [ + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, + {"type": "thinking", "thinking": "The bridge packed this one"}, + {"type": "text", "text": "The zebra owner lives in the green house."}, + ] + + class TestStripEncryptedReasoningFromInput: def test_keeps_summary_and_drops_encrypted_content_and_id(self): wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a") From a25eccafc952aa654c9d826cfff18e568c32d817 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:32:13 -0700 Subject: [PATCH 276/411] fix(azure_ai): drop the api_base path segments a relay already repeats --- .../azure_ai/passthrough/transformation.py | 18 +++++++++++++++++- ...test_azure_ai_passthrough_transformation.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index 1689755b042..ed630bc6f2b 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -62,6 +62,22 @@ def foundry_root(api_base: str) -> str: return str(url.copy_with(path="/" + "/".join(root_segments), query=None)).rstrip("/") +def without_repeated_native_prefix(root: str, native_endpoint: str) -> str: + url: Final = httpx.URL(root) + root_segments: Final = tuple(segment for segment in url.path.split("/") if segment) + native_segments: Final = tuple(segment.casefold() for segment in native_endpoint.split("/") if segment) + overlap: Final = next( + ( + length + for length in range(min(len(root_segments), len(native_segments)), 0, -1) + if tuple(segment.casefold() for segment in root_segments[-length:]) == native_segments[:length] + ), + 0, + ) + kept_segments: Final = root_segments[: len(root_segments) - overlap] + return str(url.copy_with(path="/" + "/".join(kept_segments), query=None)).rstrip("/") + + def relay_query_params( request_query_params: Mapping[str, object] | None, deployment_api_version: str | None, @@ -111,7 +127,7 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig): raise ValueError("Azure AI api base not found: set `api_base` on the deployment or AZURE_AI_API_BASE") native_endpoint: Final = strip_leading_model_segment(endpoint, (model, model_group_from(litellm_params))) - root: Final = foundry_root(base_target_url).removesuffix(f"/{native_endpoint.strip('/')}") + root: Final = without_repeated_native_prefix(foundry_root(base_target_url), native_endpoint) query_params: Final = relay_query_params( request_query_params, api_version_from(litellm_params), base_target_url ) diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index 064be954518..1b9036c50c5 100644 --- a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -135,6 +135,24 @@ def test_full_url_api_base_that_already_ends_with_the_native_path_is_not_doubled assert base == "https://my-resource.cognitiveservices.azure.com/openai/deployments/model-router" +@pytest.mark.parametrize("relayed_deployment", ["gpt-4o", "GPT-4o"]) +def test_deployment_root_api_base_is_not_repeated_when_the_relay_carries_the_deployment_path(relayed_deployment): + url, base = AzureAIPassthroughConfig().get_complete_url( + api_base="https://my-resource.openai.azure.com/openai/deployments/gpt-4o", + api_key="key", + model="gpt-4o", + endpoint=f"aoai-gpt-4o/openai/deployments/{relayed_deployment}/chat/completions", + request_query_params={"api-version": "2024-10-21"}, + litellm_params={"litellm_metadata": {"model_group": "aoai-gpt-4o"}}, + ) + + assert str(url) == ( + f"https://my-resource.openai.azure.com/openai/deployments/{relayed_deployment}/chat/completions" + "?api-version=2024-10-21" + ) + assert base == "https://my-resource.openai.azure.com" + + def test_parse_relay_under_a_models_api_base_targets_the_foundry_root(): url, _ = AzureAIPassthroughConfig().get_complete_url( api_base=f"{FOUNDRY_BASE}/models", From 708381948b0025f6f54e34b9e0e22b8d02070801 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:34:07 -0700 Subject: [PATCH 277/411] fix(cli): compare the resolved settings path when deciding whether the lite up and autoroute backups guard it --- .../proxy/client/cli/commands/claude_settings.py | 2 +- .../proxy/client/cli/test_auth_commands.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index 7e96d2a3a08..793728b0350 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -66,7 +66,7 @@ def claude_settings_path(environ: Mapping[str, str]) -> Path: def settings_file_owners(settings_path: Path) -> tuple[SettingsFileOwner, ...]: """The commands whose backups guard settings_path: `lite up` and `lite autoroute up` only ever manage the default file.""" - return SETTINGS_FILE_OWNERS if settings_path == CLAUDE_SETTINGS_PATH else () + return SETTINGS_FILE_OWNERS if settings_path.resolve() == CLAUDE_SETTINGS_PATH.resolve() else () def load_json_or_empty(path: Path) -> dict[str, JsonValue]: diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 9c3c02a5e85..d3e6d8f66ef 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -1477,6 +1477,21 @@ class TestLoginConfigClaude: assert "Run `lite down` first" in result.output assert not default_settings_path.exists() + def test_flag_refuses_while_lite_up_holds_the_default_file_reached_through_a_symlinked_config_dir(self, tmp_path): + default_config_dir = tmp_path / "default-home" / ".claude" + default_config_dir.mkdir(parents=True) + alias = tmp_path / "claude-alias" + alias.symlink_to(default_config_dir, target_is_directory=True) + (tmp_path / "claude_settings_backup.json").write_text("{}") + + result, _settings_path, _backup_path = self._run_login( + tmp_path, ["--config-claude"], config_dir_env={"CLAUDE_CONFIG_DIR": str(alias)} + ) + + assert result.exit_code != 0 + assert "`lite up` is currently managing" in result.output + assert not (default_config_dir / "settings.json").exists() + def test_flag_writes_an_alternate_config_dir_even_while_lite_up_holds_the_default_file(self, tmp_path): (tmp_path / "claude_settings_backup.json").write_text("{}") From 71cf1c7901adae5d7dd244dc98274c4ee109f149 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:34:15 -0700 Subject: [PATCH 278/411] test(e2e): drop the unused batch list client and fail loudly on a user teardown miss The failed-batch redesign left list_batches, BatchList, BatchListQuery and the batch object's metadata and created_at fields with no caller, and they duplicated the batches suite's own client. delete_user discarded its result, so a user that outlived the class fixture went unnoticed; unwrap turns that into a teardown error like delete_key already does. --- .../spend_tracking/spend_e2e_client.py | 33 ++++--------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py index 18b714d8aeb..9ac97f57f47 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py +++ b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py @@ -135,17 +135,6 @@ class BatchCreateBody(BaseModel): class BatchObject(BaseModel): id: str status: str - metadata: dict[str, str] | None = None - created_at: int | None = None - - -class BatchList(BaseModel): - data: list[BatchObject] = [] - - -class BatchListQuery(BaseModel): - model: str - limit: int class ProviderQuery(BaseModel): @@ -381,11 +370,13 @@ class SpendClient: ).user_id def delete_user(self, user_id: str) -> None: - _ = self.proxy.transport.post( - "/user/delete", - headers=self.proxy.transport.master, - json=UserDeleteBody(user_ids=[user_id]), - response_type=UserDeleteResponse, + _ = unwrap( + self.proxy.transport.post( + "/user/delete", + headers=self.proxy.transport.master, + json=UserDeleteBody(user_ids=[user_id]), + response_type=UserDeleteResponse, + ) ) def generate_key_record(self, body: KeyGenerateBody) -> KeyGenerateResponse: @@ -474,16 +465,6 @@ class SpendClient: ) ) - def list_batches(self, key: str, model: str, *, limit: int) -> list[BatchObject]: - return unwrap( - self.proxy.transport.get( - "/v1/batches", - headers=self.proxy.transport.bearer(key), - params=BatchListQuery(model=model, limit=limit), - response_type=BatchList, - ) - ).data - def retrieve_batch(self, key: str, batch_id: str, *, provider: str) -> BatchObject: return unwrap( self.proxy.transport.get( From d93baa3f2c1ca3c5123ecdee8510ee3c8d827f61 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:52:00 -0700 Subject: [PATCH 279/411] fix(bedrock): sign on a dedicated executor instead of the shared default one asyncio.to_thread puts every Bedrock signing on the loop's default executor, the same pool every provider's async entry point hops through, so signings parked on botocore's refresh lock queued unrelated providers behind Bedrock. run_aws_signing runs them on a 16-thread pool only AWS signing uses --- .../providers/bedrock_agentcore/handler.py | 6 ++--- litellm/constants.py | 1 + litellm/integrations/s3_v2.py | 6 ++--- litellm/integrations/sqs.py | 4 +-- litellm/llms/bedrock/base_aws_llm.py | 20 +++++++++++++- litellm/llms/bedrock/chat/converse_handler.py | 7 +++-- litellm/llms/bedrock/count_tokens/handler.py | 4 +-- litellm/llms/bedrock/embed/embedding.py | 7 +++-- litellm/llms/bedrock/realtime/handler.py | 6 ++--- litellm/llms/custom_httpx/llm_http_handler.py | 4 +-- .../guardrail_hooks/bedrock_guardrails.py | 10 +++---- .../llm_passthrough_endpoints.py | 9 +++---- .../llms/bedrock/test_base_aws_llm.py | 26 +++++++++++++++++++ 13 files changed, 76 insertions(+), 34 deletions(-) diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py index e01e215b5f0..306a8871b12 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py @@ -5,7 +5,6 @@ Sends JSON-RPC envelopes directly to AgentCore endpoints, bypassing the completion bridge that would otherwise strip the envelope. """ -import asyncio import json from collections.abc import AsyncIterator, Mapping from typing import Any, Final @@ -14,6 +13,7 @@ from litellm._logging import verbose_logger from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( BedrockAgentCoreA2ATransformation, ) +from litellm.llms.bedrock.base_aws_llm import run_aws_signing from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.llms.custom_http import httpxSpecialProvider @@ -46,7 +46,7 @@ class BedrockAgentCoreA2AHandler: Returns: A2A JSON-RPC response dict from the AgentCore agent """ - url, headers, body = await asyncio.to_thread( + url, headers, body = await run_aws_signing( BedrockAgentCoreA2ATransformation.get_url_and_signed_request, request_id=request_id, params=params, @@ -93,7 +93,7 @@ class BedrockAgentCoreA2AHandler: Yields: A2A streaming response events from the AgentCore agent """ - url, headers, body = await asyncio.to_thread( + url, headers, body = await run_aws_signing( BedrockAgentCoreA2ATransformation.get_url_and_signed_request, request_id=request_id, params=params, diff --git a/litellm/constants.py b/litellm/constants.py index f9389d22dea..060402cc08d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -570,6 +570,7 @@ LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS: Final = float( LOGGING_EXECUTOR_MAX_THREADS: Final = get_env_int("LOGGING_EXECUTOR_MAX_THREADS", 100) LOGGING_EXECUTOR_MAX_PENDING_TASKS: Final = get_env_int("LOGGING_EXECUTOR_MAX_PENDING_TASKS", 10_000) LOGGING_EXECUTOR_DROPPED_TASK_LOG_INTERVAL_SECONDS: Final = 30.0 +AWS_SIGNING_MAX_THREADS: Final = 16 DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE: Final = os.getenv( "DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE", "streaming.chunk.yield" ) diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index e3297cfd27c..996ff9c75af 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -24,7 +24,7 @@ from litellm.integrations.s3 import ( from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker -from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, @@ -366,7 +366,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): # Sign the request aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers) aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name) - await asyncio.to_thread(S3SigV4Auth(credentials, "s3", aws_region_name).add_auth, aws_request) + await run_aws_signing(S3SigV4Auth(credentials, "s3", aws_region_name).add_auth, aws_request) # Prepare the signed headers signed_headers: Final = dict(aws_request.headers.items()) @@ -597,7 +597,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): # Sign the request aws_request: Final = AWSRequest(method="GET", url=url, headers=headers) - await asyncio.to_thread(S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth, aws_request) + await run_aws_signing(S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth, aws_request) # Prepare the signed headers signed_headers: Final = dict(aws_request.headers.items()) diff --git a/litellm/integrations/sqs.py b/litellm/integrations/sqs.py index 31cc20f5995..d787375ca3c 100644 --- a/litellm/integrations/sqs.py +++ b/litellm/integrations/sqs.py @@ -22,7 +22,7 @@ from litellm.constants import ( SQS_SEND_MESSAGE_ACTION, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -295,7 +295,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): data=prepped.body, headers=prepped.headers, ) - await asyncio.to_thread(SigV4Auth(credentials, "sqs", self.sqs_region_name).add_auth, aws_request) + await run_aws_signing(SigV4Auth(credentials, "sqs", self.sqs_region_name).add_auth, aws_request) signed_headers: Final = dict(aws_request.headers.items()) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index eddd4d6f826..96804a1fa62 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -1,12 +1,15 @@ import asyncio import base64 +import contextvars import hashlib import json import os import re import urllib.parse from collections.abc import Callable, Mapping +from concurrent.futures import ThreadPoolExecutor from datetime import datetime +from functools import partial from threading import Lock from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, ParamSpec, TypeVar, cast, get_args, overload @@ -17,6 +20,7 @@ from litellm._logging import verbose_logger from litellm.caching.caching import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import ( + AWS_SIGNING_MAX_THREADS, BEDROCK_EMBEDDING_PROVIDERS_LITERAL, BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES, BEDROCK_IAM_CACHE_MAX_ENTRIES, @@ -1697,6 +1701,20 @@ def sign_aws_json_post( _SignParams = ParamSpec("_SignParams") _SignedRequest = TypeVar("_SignedRequest") +AWS_SIGNING_EXECUTOR: Final = ThreadPoolExecutor(max_workers=AWS_SIGNING_MAX_THREADS, thread_name_prefix="aws-signing") + + +async def run_aws_signing( + sign: Callable[_SignParams, _SignedRequest], + /, + *args: _SignParams.args, + **kwargs: _SignParams.kwargs, # kwargs-ok: ParamSpec forwarding keeps the wrapped signing signature +) -> _SignedRequest: + context: Final = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor( + AWS_SIGNING_EXECUTOR, partial(context.run, sign, *args, **kwargs) + ) + async def sign_request_off_loop_if_aws( provider_config: object, @@ -1706,5 +1724,5 @@ async def sign_request_off_loop_if_aws( **kwargs: _SignParams.kwargs, # kwargs-ok: ParamSpec forwarding keeps the wrapped sign_request signature ) -> _SignedRequest: if isinstance(provider_config, SignsRequestsWithAWS): - return await asyncio.to_thread(sign_request, *args, **kwargs) + return await run_aws_signing(sign_request, *args, **kwargs) return sign_request(*args, **kwargs) diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 3e8e1226a55..6d48ff3f07c 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -1,4 +1,3 @@ -import asyncio import json from collections.abc import Mapping from types import MappingProxyType @@ -22,7 +21,7 @@ from litellm.rust_bridge.chat_completions import rust_chat_completions_accepts from litellm.types.utils import ModelResponse from litellm.utils import CustomStreamWrapper -from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token +from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token, run_aws_signing from ..common_utils import BedrockError, _get_all_bedrock_regions, error_response_text from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call @@ -137,7 +136,7 @@ class BedrockConverseLLM(BaseAWSLLM): ) data: Final = json.dumps(request_data) - prepped: Final = await asyncio.to_thread( + prepped: Final = await run_aws_signing( self.get_request_headers, credentials=credentials, aws_region_name=litellm_params.get("aws_region_name") or "us-west-2", @@ -208,7 +207,7 @@ class BedrockConverseLLM(BaseAWSLLM): ) data: Final = json.dumps(request_data) - prepped: Final = await asyncio.to_thread( + prepped: Final = await run_aws_signing( self.get_request_headers, credentials=credentials, aws_region_name=litellm_params.get("aws_region_name") or "us-west-2", diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py index cae2315e744..d7fb510f057 100644 --- a/litellm/llms/bedrock/count_tokens/handler.py +++ b/litellm/llms/bedrock/count_tokens/handler.py @@ -4,13 +4,13 @@ AWS Bedrock CountTokens API handler. Simplified handler leveraging existing LiteLLM Bedrock infrastructure. """ -import asyncio from typing import Any, Final import httpx import litellm from litellm._logging import verbose_logger +from litellm.llms.bedrock.base_aws_llm import run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client @@ -77,7 +77,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): # Extract api_key for bearer token auth if provided api_key: Final = litellm_params.get("api_key", None) headers: Final = {"Content-Type": "application/json"} - signed_headers, signed_body = await asyncio.to_thread( + signed_headers, signed_body = await run_aws_signing( self._sign_request, service_name="bedrock", headers=headers, diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 4254d9b1d8b..be766eaedd0 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -2,7 +2,6 @@ Handles embedding calls to Bedrock's `/invoke` endpoint """ -import asyncio import copy import json import urllib.parse @@ -27,7 +26,7 @@ from litellm.types.llms.bedrock import ( ) from litellm.types.utils import EmbeddingResponse, LlmProviders -from ..base_aws_llm import AWSPreparedRequest, BaseAWSLLM, Credentials, bedrock_bearer_token +from ..base_aws_llm import AWSPreparedRequest, BaseAWSLLM, Credentials, bedrock_bearer_token, run_aws_signing from ..common_utils import BedrockError from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config @@ -357,7 +356,7 @@ class BedrockEmbedding(BaseAWSLLM): if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} - prepped = await asyncio.to_thread( + prepped = await run_aws_signing( self.get_request_headers, credentials=credentials, aws_region_name=aws_region_name, @@ -638,7 +637,7 @@ class BedrockEmbedding(BaseAWSLLM): credentials=credentials, url=status_url, headers=headers, aws_region_name=aws_region_name ) - prepped: Final = await asyncio.to_thread(sign_status_request) + prepped: Final = await run_aws_signing(sign_status_request) # LOGGING if logging_obj is not None: diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 792e3169a7a..ca2370303f2 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -21,7 +21,7 @@ from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeE from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput -from ..base_aws_llm import BaseAWSLLM +from ..base_aws_llm import BaseAWSLLM, run_aws_signing from ..common_utils import BedrockError from .transformation import BedrockRealtimeConfig @@ -149,7 +149,7 @@ class BedrockRealtime(BaseAWSLLM): verbose_proxy_logger.debug("Bedrock Realtime: Connecting to %s with model %s", endpoint_uri, model) - credentials: Final = await asyncio.to_thread( + credentials: Final = await run_aws_signing( self.get_credentials, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, @@ -170,7 +170,7 @@ class BedrockRealtime(BaseAWSLLM): "or configure credentials in the environment" ), ) - frozen_credentials: Final = await asyncio.to_thread(credentials.get_frozen_credentials) + frozen_credentials: Final = await run_aws_signing(credentials.get_frozen_credentials) # Initialize Bedrock client with aws_sdk_bedrock_runtime config: Final = Config( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3106810b4b3..13f134fce1d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -77,7 +77,7 @@ from litellm.llms.base_llm.vector_store_files.transformation import ( BaseVectorStoreFilesConfig, ) from litellm.llms.base_llm.videos.transformation import BaseVideoConfig -from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws +from litellm.llms.bedrock.base_aws_llm import run_aws_signing, sign_request_off_loop_if_aws from litellm.llms.custom_httpx.container_handler import raise_for_error_status from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -638,7 +638,7 @@ class BaseLLMHTTPHandler: headers=request_headers, ), ) - return await dispatch_async(*await asyncio.to_thread(sign_and_log, transformed)) + return await dispatch_async(*await run_aws_signing(sign_and_log, transformed)) return transform_then_dispatch() diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 0ae2c48a3b4..6a7ac4361b9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -44,7 +44,7 @@ from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicM from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, ) -from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token, run_aws_signing from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -917,7 +917,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): source, ) return BedrockGuardrailResponse() - credentials, aws_region_name = await asyncio.to_thread( + credentials, aws_region_name = await run_aws_signing( self._load_credentials, bearer_token=bedrock_bearer_token(api_key) ) allow_chunking: Final = not self._content_uses_contextual_grounding(content) @@ -1180,7 +1180,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): **base_request_data, "content": content, } # mutable-ok: outbound JSON request body - prepared_request: Final = await asyncio.to_thread( + prepared_request: Final = await run_aws_signing( self._prepare_request, credentials=credentials, data=bedrock_request_data, @@ -1878,12 +1878,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return BedrockGuardrailResponse() api_key: Final[str | None] = request_data.get("api_key") if request_data else None - credentials, aws_region_name = await asyncio.to_thread( + credentials, aws_region_name = await run_aws_signing( self._load_credentials, bearer_token=bedrock_bearer_token(api_key) ) body: Final[dict[str, object]] = {"messages": checks_messages, "checks": self.checks} - prepared_request: Final = await asyncio.to_thread( + prepared_request: Final = await run_aws_signing( self._prepare_request, credentials=credentials, data=body, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 037f663044f..bf6ee57b4fc 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -8,7 +8,6 @@ Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. from __future__ import annotations -import asyncio import hmac import inspect import json @@ -1131,7 +1130,7 @@ async def bedrock_proxy_route( ) # Add or update query parameters - from litellm.llms.bedrock.base_aws_llm import sign_aws_json_post + from litellm.llms.bedrock.base_aws_llm import run_aws_signing, sign_aws_json_post from litellm.llms.bedrock.chat import BedrockConverseLLM bedrock_llm: Final = BedrockConverseLLM() @@ -1140,7 +1139,7 @@ async def bedrock_proxy_route( data: Final = await _json_request_body(request) except Exception as e: raise HTTPException(status_code=400, detail={"error": e}) - prepped: Final = await asyncio.to_thread( + prepped: Final = await run_aws_signing( sign_aws_json_post, get_credentials=bedrock_llm.get_credentials, service_name="bedrock", @@ -1236,10 +1235,10 @@ async def comprehend_medical_proxy_route( if "stream" in data: raise HTTPException(status_code=400, detail="'stream' is not a Comprehend Medical request member") - from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, sign_aws_json_post + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing, sign_aws_json_post target_url: Final = f"https://comprehendmedical.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/" - prepped: Final = await asyncio.to_thread( + prepped: Final = await run_aws_signing( sign_aws_json_post, get_credentials=partial(BaseAWSLLM().get_credentials, aws_region_name=aws_region_name), service_name="comprehendmedical", diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index ca93f58f366..e24ea1ca611 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -1,5 +1,6 @@ import asyncio import json +from concurrent.futures import ThreadPoolExecutor import os import threading import time @@ -23,6 +24,7 @@ from litellm.llms.bedrock.base_aws_llm import ( AwsAuthError, BaseAWSLLM, Boto3CredentialsInfo, + run_aws_signing, sign_request_off_loop_if_aws, ) from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe @@ -3247,3 +3249,27 @@ async def test_sign_request_off_loop_if_aws_keeps_the_loop_serving_while_credent assert "Authorization" in signed assert probe.served_during_refresh is True + + +async def test_run_aws_signing_leaves_the_default_executor_free_for_other_providers(): + """A signing parked on botocore's refresh lock must not hold a default-executor thread, since every + other provider's async entry point hops through that same executor.""" + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + signing_parked = asyncio.Event() + refresh_done = threading.Event() + + def sign() -> str: + loop.call_soon_threadsafe(signing_parked.set) + refresh_done.wait() + return threading.current_thread().name + + signing = asyncio.create_task(run_aws_signing(sign)) + try: + await asyncio.wait_for(signing_parked.wait(), timeout=5) + other_provider = await asyncio.wait_for(loop.run_in_executor(None, threading.current_thread), timeout=5) + finally: + refresh_done.set() + + assert other_provider.name != await signing + assert (await signing).startswith("aws-signing") From 9c6a385efe1f61cbd9d276ed09427102aec820ff Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:57:37 -0700 Subject: [PATCH 280/411] docs(azure_ai): drop size from the MAI image edit router docstring --- litellm/llms/azure_ai/image_edit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/azure_ai/image_edit/__init__.py b/litellm/llms/azure_ai/image_edit/__init__.py index 51a23859058..fda9335a5d6 100644 --- a/litellm/llms/azure_ai/image_edit/__init__.py +++ b/litellm/llms/azure_ai/image_edit/__init__.py @@ -23,7 +23,7 @@ def get_azure_ai_image_edit_config(model: str) -> BaseImageEditConfig: """ Get the appropriate image edit config for an Azure AI model. - - MAI models use /mai/v1/images/edits with multipart form data and size + - MAI models use /mai/v1/images/edits with multipart form data - FLUX 2 models use JSON with base64 image - FLUX 1 models use multipart/form-data """ From 8627576c9c37bc475ac35d98ab6f44d1ea8978d9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:11:04 -0700 Subject: [PATCH 281/411] fix(azure_ai): keep a deployment named like the first native path segment in the relayed URL --- .../llms/azure_ai/passthrough/transformation.py | 5 +++++ .../test_azure_ai_passthrough_transformation.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index ed630bc6f2b..f2be1d95593 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -62,6 +62,10 @@ def foundry_root(api_base: str) -> str: return str(url.copy_with(path="/" + "/".join(root_segments), query=None)).rstrip("/") +def is_repeated_native_prefix(native_segments: tuple[str, ...], overlap: int) -> bool: + return overlap == len(native_segments) or native_segments[0] == "openai" + + def without_repeated_native_prefix(root: str, native_endpoint: str) -> str: url: Final = httpx.URL(root) root_segments: Final = tuple(segment for segment in url.path.split("/") if segment) @@ -71,6 +75,7 @@ def without_repeated_native_prefix(root: str, native_endpoint: str) -> str: length for length in range(min(len(root_segments), len(native_segments)), 0, -1) if tuple(segment.casefold() for segment in root_segments[-length:]) == native_segments[:length] + and is_repeated_native_prefix(native_segments, length) ), 0, ) diff --git a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index 1b9036c50c5..c8007acf70f 100644 --- a/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -153,6 +153,20 @@ def test_deployment_root_api_base_is_not_repeated_when_the_relay_carries_the_dep assert base == "https://my-resource.openai.azure.com" +def test_deployment_named_like_the_first_native_segment_keeps_its_deployment_root(): + url, base = AzureAIPassthroughConfig().get_complete_url( + api_base="https://my-resource.openai.azure.com/openai/deployments/chat", + api_key="key", + model="chat", + endpoint="aoai-chat/chat/completions", + request_query_params={"api-version": "2024-10-21"}, + litellm_params={"litellm_metadata": {"model_group": "aoai-chat"}}, + ) + + assert str(url) == "https://my-resource.openai.azure.com/openai/deployments/chat/chat/completions?api-version=2024-10-21" + assert base == "https://my-resource.openai.azure.com/openai/deployments/chat" + + def test_parse_relay_under_a_models_api_base_targets_the_foundry_root(): url, _ = AzureAIPassthroughConfig().get_complete_url( api_base=f"{FOUNDRY_BASE}/models", From 3dcc09b26ccb0aa781e7bb62007e73e93c2bd9f6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:16:11 -0700 Subject: [PATCH 282/411] fix(bedrock): keep the signing pool to AWS configs on the async-transform path The async-transform path in the shared handler sent every provider's sign_and_log to the AWS pool, so Ollama, Snowflake, and watsonx queued behind Bedrock refreshes there. Only SignsRequestsWithAWS configs take run_aws_signing now, the rest keep the default-executor hop they had. The executor isolation test also runs on its own loop instead of pinning a one-thread default executor on the session-scoped pytest loop --- litellm/llms/custom_httpx/llm_http_handler.py | 9 +++- .../llms/bedrock/test_base_aws_llm.py | 41 +++++++++++-------- .../custom_httpx/test_llm_http_handler.py | 21 ++++++++++ 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 13f134fce1d..e720428847d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -77,7 +77,7 @@ from litellm.llms.base_llm.vector_store_files.transformation import ( BaseVectorStoreFilesConfig, ) from litellm.llms.base_llm.videos.transformation import BaseVideoConfig -from litellm.llms.bedrock.base_aws_llm import run_aws_signing, sign_request_off_loop_if_aws +from litellm.llms.bedrock.base_aws_llm import SignsRequestsWithAWS, run_aws_signing, sign_request_off_loop_if_aws from litellm.llms.custom_httpx.container_handler import raise_for_error_status from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -638,7 +638,12 @@ class BaseLLMHTTPHandler: headers=request_headers, ), ) - return await dispatch_async(*await run_aws_signing(sign_and_log, transformed)) + signed_request: Final = await ( + run_aws_signing(sign_and_log, transformed) + if isinstance(provider_config, SignsRequestsWithAWS) + else asyncio.to_thread(sign_and_log, transformed) + ) + return await dispatch_async(*signed_request) return transform_then_dispatch() diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index e24ea1ca611..2c7e476e9a8 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -3251,25 +3251,30 @@ async def test_sign_request_off_loop_if_aws_keeps_the_loop_serving_while_credent assert probe.served_during_refresh is True -async def test_run_aws_signing_leaves_the_default_executor_free_for_other_providers(): +def test_run_aws_signing_leaves_the_default_executor_free_for_other_providers(): """A signing parked on botocore's refresh lock must not hold a default-executor thread, since every - other provider's async entry point hops through that same executor.""" - loop = asyncio.get_running_loop() - loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) - signing_parked = asyncio.Event() - refresh_done = threading.Event() + other provider's async entry point hops through that same executor. The scenario runs on its own loop + so the one-thread default executor it pins never leaks into the session loop.""" - def sign() -> str: - loop.call_soon_threadsafe(signing_parked.set) - refresh_done.wait() - return threading.current_thread().name + async def scenario() -> tuple[str, str]: + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + signing_parked = asyncio.Event() + refresh_done = threading.Event() - signing = asyncio.create_task(run_aws_signing(sign)) - try: - await asyncio.wait_for(signing_parked.wait(), timeout=5) - other_provider = await asyncio.wait_for(loop.run_in_executor(None, threading.current_thread), timeout=5) - finally: - refresh_done.set() + def sign() -> str: + loop.call_soon_threadsafe(signing_parked.set) + refresh_done.wait() + return threading.current_thread().name - assert other_provider.name != await signing - assert (await signing).startswith("aws-signing") + signing = asyncio.create_task(run_aws_signing(sign)) + try: + await asyncio.wait_for(signing_parked.wait(), timeout=5) + other_provider = await asyncio.wait_for(loop.run_in_executor(None, threading.current_thread), timeout=5) + finally: + refresh_done.set() + return other_provider.name, await signing + + other_provider, signing_thread = asyncio.run(scenario()) + assert other_provider != signing_thread + assert signing_thread.startswith("aws-signing") diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 1140090f9cf..cea2d439198 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -20,6 +20,7 @@ from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, ) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.bedrock.base_aws_llm import SignsRequestsWithAWS from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import ( @@ -3431,6 +3432,26 @@ async def test_completion_signs_and_logs_off_the_event_loop_after_the_async_tran assert captured["body"] == {"transformed_by": "async"} assert config.sign_threads and all(thread is not loop_thread for thread in config.sign_threads) assert pre_call_threads and all(thread is not loop_thread for thread in pre_call_threads) + assert not any(thread.name.startswith("aws-signing") for thread in config.sign_threads + pre_call_threads) + + +class _AWSTransformRecordingConfig(SignsRequestsWithAWS, _TransformRecordingConfig): + pass + + +async def test_completion_signs_aws_configs_on_the_aws_signing_pool_after_the_async_transform(): + config = _AWSTransformRecordingConfig(transform_async=True) + pre_call_threads = [] + logging_obj = Mock(dynamic_success_callbacks=None, model_call_details={}) + logging_obj.pre_call.side_effect = lambda **kwargs: pre_call_threads.append(threading.current_thread()) + + pending, captured = _start_async_completion(config, logging_obj) + response = await pending + + assert response.choices[0].message.content == "async" + assert captured["body"] == {"transformed_by": "async"} + assert config.sign_threads and all(thread.name.startswith("aws-signing") for thread in config.sign_threads) + assert pre_call_threads and all(thread.name.startswith("aws-signing") for thread in pre_call_threads) async def test_completion_keeps_sync_transform_request_before_returning_by_default(): From 5a1be5642606eb56945e0dfd91ca02647a592a52 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:27:54 -0700 Subject: [PATCH 283/411] fix(router): drop bridge-tagged reasoning blocks whole when the minting deployment is not in the routed group The cross-group branch of EncryptedContentAffinityCheck removed only the signature from Anthropic-shaped thinking blocks, which left unsigned thinking blocks that Anthropic and Bedrock reject (thinking.signature: Field required). Drop the whole block, the way #40280 drops undecryptable Responses input items, so the routed request carries the conversation text with no reasoning item for those turns --- .../prompt_templates/common_utils.py | 25 +++++++------------ ...ore_utils_prompt_templates_common_utils.py | 4 +-- .../test_encrypted_content_affinity_check.py | 8 +++--- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 7ca2a8aad6a..d70530534da 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1890,8 +1890,13 @@ def is_encrypted_reasoning_block(block: object) -> bool: def strip_encrypted_reasoning_from_messages(messages: object) -> None: - """Drop the encrypted reasoning a routed deployment cannot decrypt from Anthropic-shaped - history, keeping the readable thinking text. + """Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from + Anthropic-shaped history. + + The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a + provider that did not mint the block rejects it signed (a foreign signature) and unsigned + (a missing signature) alike, so keeping its text as an unsigned thinking block only moves + the 400 from the router to the provider. Mutates the content lists in place: the router's fallback snapshot shares these message objects, so a rebound list would replay the stripped blocks on the fallback hop. @@ -1914,20 +1919,8 @@ def _anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: def _strip_encrypted_reasoning_from_blocks(content: object) -> None: blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance - stripped: Final = tuple(_without_encrypted_reasoning_block(block) for block in blocks) - blocks[:] = (block for block in stripped if block is not None) # rebind-ok: list shared with fallback snapshot - - -def _without_encrypted_reasoning_block(block: object) -> object | None: - if not is_encrypted_reasoning_block(block): - return block - mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by is_encrypted_reasoning_block - if mapping.get("type") != "thinking" or not mapping.get("thinking"): - return None - kept: Final[dict[str, object]] = { # mutable-ok: thinking block rebuilt without the undecryptable signature - key: value for key, value in mapping.items() if key != "signature" - } - return kept + kept: Final = tuple(block for block in blocks if not is_encrypted_reasoning_block(block)) + blocks[:] = kept # rebind-ok: shared with fallback snapshot def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 71b71d55fa7..7c1445d79c9 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1633,7 +1633,7 @@ class TestEncryptedReasoningReplay: def test_is_encrypted_reasoning_block(self, block, expected): assert is_encrypted_reasoning_block(block) is expected - def test_strip_keeps_the_readable_thinking_and_drops_the_undecryptable_bytes(self): + def test_strip_drops_every_bridge_tagged_block_and_leaves_no_unsigned_thinking_behind(self): assistant_content = [ {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, {"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")}, @@ -1652,9 +1652,9 @@ class TestEncryptedReasoningReplay: assert messages[1]["content"] is assistant_content assert assistant_content == [ {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, - {"type": "thinking", "thinking": "packed by the bridge"}, {"type": "text", "text": "answer"}, ] + assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") assert messages[0] == {"role": "user", "content": "question"} assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]} diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 57c02e43fc1..ea8e2eacaa6 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1635,8 +1635,10 @@ def _bridge_replayed_anthropic_messages(minted_by: str) -> list: async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_routed_to_another_group(): """ The /v1/messages twin of the tier-change case: the routed group holds no deployment - of the org that minted the reasoning, so the bridge-tagged blocks are stripped down - to their readable thinking text and the request dispatches to the routed pool. + of the org that minted the reasoning, so the bridge-tagged blocks are dropped whole + and the request dispatches to the routed pool. No unsigned thinking block may be left + behind: Anthropic and Bedrock reject a thinking block with a missing signature the + same way they reject a foreign one. """ originating = _make_originating_mock(None, "key-a", model_name="gpt-reasoning-tier") mock_router = _make_router_mock_with_cooldown( @@ -1660,9 +1662,9 @@ async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_ assert messages[1]["content"] is assistant_content assert assistant_content == [ {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, - {"type": "thinking", "thinking": "The bridge packed this one"}, {"type": "text", "text": "The zebra owner lives in the green house."}, ] + assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") class TestStripEncryptedReasoningFromInput: From b64197c2898dc6db980ec5b944ce6189caf3ae84 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:44:36 -0700 Subject: [PATCH 284/411] fix(terraform): preserve undeclared server-side key metadata on update (#40514) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/litellm/client.go | 14 +- terraform/provider/litellm/resource_key.go | 59 +++++++- .../provider/litellm/resource_key_test.go | 133 ++++++++++++++++++ 3 files changed, 202 insertions(+), 4 deletions(-) diff --git a/terraform/provider/litellm/client.go b/terraform/provider/litellm/client.go index 0f825d85d31..0dfbe3dd3c9 100644 --- a/terraform/provider/litellm/client.go +++ b/terraform/provider/litellm/client.go @@ -80,16 +80,24 @@ func (c *Client) UpdateKey(key *Key) (*Key, error) { updateData := map[string]interface{}{ "key": key.Key, "team_id": key.TeamID, - "metadata": key.Metadata, "key_alias": key.KeyAlias, "aliases": key.Aliases, "permissions": key.Permissions, "model_max_budget": key.ModelMaxBudget, - "model_rpm_limit": key.ModelRPMLimit, - "model_tpm_limit": key.ModelTPMLimit, "blocked": key.Blocked, } + // The proxy keeps the stored metadata only when the field is absent, so nil means omit. + if key.Metadata != nil { + updateData["metadata"] = key.Metadata + } + if key.ModelRPMLimit != nil { + updateData["model_rpm_limit"] = key.ModelRPMLimit + } + if key.ModelTPMLimit != nil { + updateData["model_tpm_limit"] = key.ModelTPMLimit + } + // The proxy rejects an empty-string budget_duration with a 400, so only // send it when set. if key.BudgetDuration != "" { diff --git a/terraform/provider/litellm/resource_key.go b/terraform/provider/litellm/resource_key.go index 0d8674f2d4c..a37041fe7e8 100644 --- a/terraform/provider/litellm/resource_key.go +++ b/terraform/provider/litellm/resource_key.go @@ -223,6 +223,7 @@ func resourceKeyRead(ctx context.Context, d *schema.ResourceData, m interface{}) return nil } + key.Metadata = declaredKeyMetadata(key.Metadata, d.Get("metadata").(map[string]interface{})) mapKeyToResourceData(d, key) return nil } @@ -232,15 +233,71 @@ func resourceKeyUpdate(ctx context.Context, d *schema.ResourceData, m interface{ key := &Key{Key: d.Id()} mapResourceDataToKey(d, key) + key.ModelRPMLimit = changedMap(d, "model_rpm_limit") + key.ModelTPMLimit = changedMap(d, "model_tpm_limit") - _, err := c.UpdateKey(key) + metadata, err := plannedKeyMetadata(c, d) if err != nil { return diag.FromErr(fmt.Errorf("error updating key: %s", err)) } + key.Metadata = metadata + + if _, err := c.UpdateKey(key); err != nil { + return diag.FromErr(fmt.Errorf("error updating key: %s", err)) + } return resourceKeyRead(ctx, d, m) } +func changedMap(d *schema.ResourceData, name string) map[string]interface{} { + if !d.HasChange(name) { + return nil + } + return d.Get(name).(map[string]interface{}) +} + +func plannedKeyMetadata(c *Client, d *schema.ResourceData) (map[string]interface{}, error) { + if !d.HasChange("metadata") { + return nil, nil + } + current, err := c.GetKey(d.Id()) + if err != nil { + return nil, err + } + if current == nil { + return nil, fmt.Errorf("key %s no longer exists", d.Id()) + } + oldDeclared, newDeclared := d.GetChange("metadata") + return mergeKeyMetadata(current.Metadata, oldDeclared.(map[string]interface{}), newDeclared.(map[string]interface{})), nil +} + +func declaredKeyMetadata(server, declared map[string]interface{}) map[string]interface{} { + if server == nil { + return nil + } + result := make(map[string]interface{}, len(declared)) + for k := range declared { + if v, ok := server[k]; ok { + result[k] = v + } + } + return result +} + +func mergeKeyMetadata(server, oldDeclared, newDeclared map[string]interface{}) map[string]interface{} { + result := make(map[string]interface{}, len(server)+len(newDeclared)) + for k, v := range server { + result[k] = v + } + for k := range oldDeclared { + delete(result, k) + } + for k, v := range newDeclared { + result[k] = v + } + return result +} + func resourceKeyDelete(ctx context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics { c := m.(*Client) diff --git a/terraform/provider/litellm/resource_key_test.go b/terraform/provider/litellm/resource_key_test.go index 91f0061a9ef..d8f3cfe86b3 100644 --- a/terraform/provider/litellm/resource_key_test.go +++ b/terraform/provider/litellm/resource_key_test.go @@ -6,9 +6,11 @@ import ( "io" "net/http" "net/http/httptest" + "reflect" "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) func newKeyResourceData(t *testing.T, raw map[string]interface{}) *schema.ResourceData { @@ -254,3 +256,134 @@ func TestGetKeyUnwrapsInfoEnvelope(t *testing.T) { t.Errorf("RPMLimit not parsed: %+v", key.RPMLimit) } } + +// fakeKeyProxy serves /key/info from stored metadata and applies /key/update +// the way the proxy does: an absent "metadata" keeps the stored map, a +// present one replaces it wholesale. +type fakeKeyProxy struct { + metadata map[string]interface{} + updates []map[string]interface{} +} + +func (p *fakeKeyProxy) handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/key/info": + json.NewEncoder(w).Encode(map[string]interface{}{ + "key": "hash-1", + "info": map[string]interface{}{"key_alias": "alias-1", "models": []string{"gpt-4o-mini"}, "metadata": p.metadata}, + }) + case "/key/update": + var body map[string]interface{} + json.NewDecoder(r.Body).Decode(&body) + p.updates = append(p.updates, body) + if m, ok := body["metadata"].(map[string]interface{}); ok { + p.metadata = m + } + json.NewEncoder(w).Encode(map[string]interface{}{"key": "hash-1", "metadata": p.metadata}) + default: + http.NotFound(w, r) + } + } +} + +func applyKeyUpdate(t *testing.T, client *Client, stateAttrs map[string]string, config map[string]interface{}) *terraform.InstanceState { + t.Helper() + r := resourceKey() + state := &terraform.InstanceState{ID: "hash-1", Attributes: stateAttrs} + diff, err := r.Diff(context.Background(), state, terraform.NewResourceConfigRaw(config), client) + if err != nil { + t.Fatalf("Diff returned error: %v", err) + } + if diff == nil { + t.Fatalf("expected a non-empty diff between %v and %v", stateAttrs, config) + } + newState, diags := r.Apply(context.Background(), state, diff, client) + if diags.HasError() { + t.Fatalf("Apply returned error: %v", diags) + } + return newState +} + +func TestKeyUpdateWithoutMetadataChangePreservesServerMetadata(t *testing.T) { + proxy := &fakeKeyProxy{metadata: map[string]interface{}{"a": "1", "server_side": "x", "model_rpm_limit": map[string]interface{}{"gpt-4o-mini": float64(5)}}} + srv := httptest.NewServer(proxy.handler()) + defer srv.Close() + client := NewClient(srv.URL, "test-key", true) + + newState := applyKeyUpdate(t, client, + map[string]string{"key_alias": "alias-1", "max_budget": "10", "metadata.%": "1", "metadata.a": "1"}, + map[string]interface{}{"key_alias": "alias-1", "max_budget": 20, "metadata": map[string]interface{}{"a": "1"}}, + ) + + if len(proxy.updates) != 1 { + t.Fatalf("expected one /key/update call, got %d", len(proxy.updates)) + } + for _, field := range []string{"metadata", "model_rpm_limit", "model_tpm_limit"} { + if _, present := proxy.updates[0][field]; present { + t.Errorf("unchanged %q was sent on /key/update: %v", field, proxy.updates[0][field]) + } + } + if proxy.metadata["server_side"] != "x" { + t.Errorf("server-side metadata lost: %v", proxy.metadata) + } + if got := newState.Attributes["metadata.%"]; got != "1" { + t.Errorf("state metadata should hold only the declared entry, got %v", newState.Attributes) + } + if got := newState.Attributes["metadata.a"]; got != "1" { + t.Errorf("metadata.a = %q, want 1", got) + } +} + +func TestKeyUpdateWithMetadataChangeMergesOverServerMetadata(t *testing.T) { + proxy := &fakeKeyProxy{metadata: map[string]interface{}{"a": "1", "b": "2", "server_side": "x"}} + srv := httptest.NewServer(proxy.handler()) + defer srv.Close() + client := NewClient(srv.URL, "test-key", true) + + applyKeyUpdate(t, client, + map[string]string{"key_alias": "alias-1", "metadata.%": "2", "metadata.a": "1", "metadata.b": "2"}, + map[string]interface{}{"key_alias": "alias-1", "metadata": map[string]interface{}{"a": "2", "c": "3"}}, + ) + + want := map[string]interface{}{"a": "2", "c": "3", "server_side": "x"} + if !reflect.DeepEqual(proxy.metadata, want) { + t.Errorf("metadata after update = %v, want %v", proxy.metadata, want) + } +} + +func TestKeyUpdateSendsChangedModelLimits(t *testing.T) { + proxy := &fakeKeyProxy{metadata: map[string]interface{}{}} + srv := httptest.NewServer(proxy.handler()) + defer srv.Close() + client := NewClient(srv.URL, "test-key", true) + + applyKeyUpdate(t, client, + map[string]string{"key_alias": "alias-1", "model_rpm_limit.%": "1", "model_rpm_limit.gpt-4o-mini": "5"}, + map[string]interface{}{"key_alias": "alias-1", "model_rpm_limit": map[string]interface{}{"gpt-4o-mini": 7}}, + ) + + got, ok := proxy.updates[0]["model_rpm_limit"].(map[string]interface{}) + if !ok || got["gpt-4o-mini"] != float64(7) { + t.Errorf("changed model_rpm_limit not sent: %v", proxy.updates[0]) + } +} + +func TestKeyReadKeepsOnlyDeclaredMetadata(t *testing.T) { + proxy := &fakeKeyProxy{metadata: map[string]interface{}{"a": "1", "server_side": "x"}} + srv := httptest.NewServer(proxy.handler()) + defer srv.Close() + client := NewClient(srv.URL, "test-key", true) + + d := newKeyResourceData(t, map[string]interface{}{"metadata": map[string]interface{}{"a": "1"}}) + d.SetId("hash-1") + if diags := resourceKeyRead(context.Background(), d, client); diags.HasError() { + t.Fatalf("Read returned error: %v", diags) + } + + want := map[string]interface{}{"a": "1"} + if got := d.Get("metadata"); !reflect.DeepEqual(got, want) { + t.Errorf("metadata in state = %v, want %v", got, want) + } +} From 4ddc5e2c29fe28665888a01b7fb53bd3f6889362 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:44:50 -0700 Subject: [PATCH 285/411] fix(terraform): read team per-model rpm/tpm limits from metadata and clear them on removal (#40439) * fix(terraform): read team per-model rpm/tpm limits from metadata and clear them on removal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(terraform): drop explanatory comments from team per-model limit helper and tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/litellm/resource_team.go | 25 +++++-- .../provider/litellm/resource_team_test.go | 71 +++++++++++++++++++ 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/terraform/provider/litellm/resource_team.go b/terraform/provider/litellm/resource_team.go index 24c47843cd1..cea0344b72e 100644 --- a/terraform/provider/litellm/resource_team.go +++ b/terraform/provider/litellm/resource_team.go @@ -263,11 +263,11 @@ func resourceLiteLLMTeamRead(d *schema.ResourceData, m interface{}) error { d.Set("team_member_tpm_limit", *teamResp.TeamMemberTPMLimit) } d.Set("team_member_key_duration", GetStringValue(teamResp.TeamMemberKeyDuration, d.Get("team_member_key_duration").(string))) - if teamResp.ModelRPMLimit != nil { - d.Set("model_rpm_limit", teamResp.ModelRPMLimit) + if v := teamModelLimit(teamResp.ModelRPMLimit, teamResp.Metadata, "model_rpm_limit"); v != nil { + d.Set("model_rpm_limit", v) } - if teamResp.ModelTPMLimit != nil { - d.Set("model_tpm_limit", teamResp.ModelTPMLimit) + if v := teamModelLimit(teamResp.ModelTPMLimit, teamResp.Metadata, "model_tpm_limit"); v != nil { + d.Set("model_tpm_limit", v) } if teamResp.AllowedPassthroughRoutes != nil { d.Set("allowed_passthrough_routes", teamResp.AllowedPassthroughRoutes) @@ -364,14 +364,19 @@ func buildTeamData(d *schema.ResourceData, teamID string) map[string]interface{} "organization_id", "tpm_limit", "rpm_limit", "max_budget", "budget_duration", "models", "blocked", "team_member_permissions", "model_aliases", "guardrails", "prompts", "team_member_budget", "team_member_budget_duration", "team_member_rpm_limit", - "team_member_tpm_limit", "team_member_key_duration", "model_rpm_limit", - "model_tpm_limit", "allowed_passthrough_routes", + "team_member_tpm_limit", "team_member_key_duration", "allowed_passthrough_routes", } { if v, ok := d.GetOk(key); ok { teamData[key] = v } } + for _, key := range []string{"model_rpm_limit", "model_tpm_limit"} { + if v, ok := d.GetOk(key); ok || d.HasChange(key) { + teamData[key] = v + } + } + if v, ok := d.GetOk("soft_budget"); ok { teamData["soft_budget"] = v } else if d.HasChange("soft_budget") { @@ -404,6 +409,14 @@ func buildTeamMetadata(d *schema.ResourceData) map[string]interface{} { return metadata } +func teamModelLimit(topLevel, metadata map[string]interface{}, key string) map[string]interface{} { + if topLevel != nil { + return topLevel + } + nested, _ := metadata[key].(map[string]interface{}) + return nested +} + func splitTeamMetadata(raw map[string]interface{}) (map[string]string, []string, []string) { metadata := map[string]string{} var tags, alertEmails []string diff --git a/terraform/provider/litellm/resource_team_test.go b/terraform/provider/litellm/resource_team_test.go index 9638378cdfe..ca4114ad24f 100644 --- a/terraform/provider/litellm/resource_team_test.go +++ b/terraform/provider/litellm/resource_team_test.go @@ -250,6 +250,77 @@ func TestTeamReadMapsNewFields(t *testing.T) { } } +func TestTeamReadMapsPerModelLimitsFromMetadata(t *testing.T) { + var captured map[string]interface{} + srv := newTeamTestServer(t, &captured, `{ + "team_id": "team-1", + "team_info": { + "team_id": "team-1", + "team_alias": "eng", + "model_rpm_limit": null, + "model_tpm_limit": null, + "metadata": { + "department": "eng", + "model_rpm_limit": {"gpt-4o-mini": 250}, + "model_tpm_limit": {"gpt-4o-mini": 5000} + } + } + }`) + defer srv.Close() + + d := newTeamResourceData(t, map[string]interface{}{ + "team_alias": "eng", + "model_rpm_limit": map[string]interface{}{"gpt-4o-mini": 100}, + }) + d.SetId("team-1") + + if err := resourceLiteLLMTeamRead(d, NewClient(srv.URL, "test-key", true)); err != nil { + t.Fatalf("read returned error: %v", err) + } + if got := d.Get("model_rpm_limit"); !reflect.DeepEqual(got, map[string]interface{}{"gpt-4o-mini": 250}) { + t.Errorf("model_rpm_limit = %v, want server value 250", got) + } + if got := d.Get("model_tpm_limit"); !reflect.DeepEqual(got, map[string]interface{}{"gpt-4o-mini": 5000}) { + t.Errorf("model_tpm_limit = %v, want server value 5000", got) + } + if got := d.Get("metadata"); !reflect.DeepEqual(got, map[string]interface{}{"department": "eng"}) { + t.Errorf("metadata = %v, want per-model limits kept out of the string map", got) + } +} + +func TestTeamUpdateClearsRemovedPerModelLimits(t *testing.T) { + var captured map[string]interface{} + srv := newTeamTestServer(t, &captured, `{"team_id":"team-1","team_info":{"team_id":"team-1","team_alias":"eng"}}`) + defer srv.Close() + + res := ResourceLiteLLMTeam() + priorData := schema.TestResourceDataRaw(t, res.Schema, map[string]interface{}{ + "team_alias": "eng", + "model_rpm_limit": map[string]interface{}{"gpt-4o-mini": 100}, + "model_tpm_limit": map[string]interface{}{"gpt-4o-mini": 5000}, + }) + priorData.SetId("team-1") + prior := priorData.State() + config := terraform.NewResourceConfigRaw(map[string]interface{}{"team_alias": "eng"}) + diff, err := res.Diff(context.Background(), prior, config, nil) + if err != nil { + t.Fatalf("diff failed: %v", err) + } + d, err := schema.InternalMap(res.Schema).Data(prior, diff) + if err != nil { + t.Fatalf("data failed: %v", err) + } + + if err := resourceLiteLLMTeamUpdate(d, NewClient(srv.URL, "test-key", true)); err != nil { + t.Fatalf("update failed: %v", err) + } + for _, k := range []string{"model_rpm_limit", "model_tpm_limit"} { + if got, ok := captured[k]; !ok || !reflect.DeepEqual(got, map[string]interface{}{}) { + t.Errorf("payload %s = %v (present=%v), want explicit empty map", k, got, ok) + } + } +} + // rpm_limit_type / tpm_limit_type are accepted by /team/new but not // /team/update, so create must send them and update must not. func TestTeamLimitTypesSentOnCreateOnly(t *testing.T) { From 18eb6d799880f4fc4f4ea79bdbce6b7839530625 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:45:02 -0700 Subject: [PATCH 286/411] feat(terraform): allow custom team_id on litellm_team (#40459) Add an optional, computed, force-new team_id argument to the litellm_team resource. When set, it is sent to /team/new as the team_id; when omitted the provider keeps generating a UUID. Read mirrors the resource id into state so import and plan stay clean. Resolves LIT-6399 Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/CHANGELOG.md | 1 + terraform/provider/docs/resources/team.md | 16 +++- terraform/provider/litellm/resource_team.go | 17 +++- .../provider/litellm/resource_team_test.go | 82 +++++++++++++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index 842bfb4bdb1..de54207ea5a 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -16,6 +16,7 @@ longer signal it. ### Added +- **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement - **jwt_key_mapping**: New `litellm_jwt_key_mapping` resource for the proxy's JWT to virtual key mappings, so JWT clients identified by a claim (`client_id`, `azp`, `sub`) map to virtual keys and inherit their models, budgets and rate limits. Supports `description` and `is_active`, rotating the mapped key in place, and forces replacement when the claim name or value changes - **team**: `soft_budget`, `tags`, and `soft_budget_alerting_emails` attributes on `litellm_team`, matching what `/team/new` and `/team/update` already accept; `soft_budget_alerting_emails` is sent under `metadata`, where the proxy reads it - **user**: New `litellm_user` resource and `litellm_user` / `litellm_users` data sources for managing internal users diff --git a/terraform/provider/docs/resources/team.md b/terraform/provider/docs/resources/team.md index 821d8c1dee3..19575907269 100644 --- a/terraform/provider/docs/resources/team.md +++ b/terraform/provider/docs/resources/team.md @@ -14,6 +14,16 @@ resource "litellm_team" "engineering" { } ``` +### Team with a Custom ID + +```hcl +resource "litellm_team" "platform" { + team_id = "platform-team" + team_alias = "platform" + models = ["gpt-4-proxy"] +} +``` + ### Team with Comprehensive Configuration ```hcl @@ -92,6 +102,8 @@ resource "litellm_team" "model_dependent_team" { The following arguments are supported: +* `team_id` - (Optional) A stable, human-readable ID for the team (for example `platform-team`). If omitted, the provider generates a random UUID. Changing this forces a new team to be created. + * `team_alias` - (Required) A human-readable identifier for the team. * `organization_id` - (Optional) The ID of the organization this team belongs to. @@ -152,7 +164,7 @@ The following arguments are supported: In addition to the arguments above, the following attributes are exported: -* `id` - The unique identifier for the team. +* `id` - The unique identifier for the team, equal to `team_id`. ## Import @@ -162,7 +174,7 @@ Teams can be imported using the team ID: terraform import litellm_team.engineering ``` -Note: The team ID is generated when the team is created and is different from the `team_alias`. +Note: Unless `team_id` is set, the team ID is generated when the team is created and is different from the `team_alias`. ## Note on Team Members diff --git a/terraform/provider/litellm/resource_team.go b/terraform/provider/litellm/resource_team.go index cea0344b72e..bf7d2508077 100644 --- a/terraform/provider/litellm/resource_team.go +++ b/terraform/provider/litellm/resource_team.go @@ -31,6 +31,13 @@ func ResourceLiteLLMTeam() *schema.Resource { }, Schema: map[string]*schema.Schema{ + "team_id": { + Type: schema.TypeString, + Optional: true, + Computed: true, + ForceNew: true, + Description: "Unique ID for the team. Generated by the provider if not provided", + }, "team_alias": { Type: schema.TypeString, Required: true, @@ -162,7 +169,7 @@ func ResourceLiteLLMTeam() *schema.Resource { func resourceLiteLLMTeamCreate(d *schema.ResourceData, m interface{}) error { client := m.(*Client) - teamID := uuid.New().String() + teamID := resolveTeamID(d) teamData := buildTeamData(d, teamID) // Throughput limit types are only accepted by /team/new, not /team/update. @@ -214,6 +221,7 @@ func resourceLiteLLMTeamRead(d *schema.ResourceData, m interface{}) error { teamResp := infoResp.TeamInfo // Update the state with values from the response or fall back to the data passed in during creation + d.Set("team_id", d.Id()) d.Set("team_alias", GetStringValue(teamResp.TeamAlias, d.Get("team_alias").(string))) d.Set("organization_id", GetStringValue(teamResp.OrganizationID, d.Get("organization_id").(string))) @@ -354,6 +362,13 @@ func resourceLiteLLMTeamDelete(d *schema.ResourceData, m interface{}) error { return nil } +func resolveTeamID(d *schema.ResourceData) string { + if v, ok := d.GetOk("team_id"); ok { + return v.(string) + } + return uuid.New().String() +} + func buildTeamData(d *schema.ResourceData, teamID string) map[string]interface{} { teamData := map[string]interface{}{ "team_id": teamID, diff --git a/terraform/provider/litellm/resource_team_test.go b/terraform/provider/litellm/resource_team_test.go index ca4114ad24f..35d60401d30 100644 --- a/terraform/provider/litellm/resource_team_test.go +++ b/terraform/provider/litellm/resource_team_test.go @@ -9,6 +9,7 @@ import ( "reflect" "testing" + "github.com/google/uuid" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) @@ -85,6 +86,87 @@ func TestTeamCreateSendsSoftBudgetTagsAndAlertEmails(t *testing.T) { } } +func TestTeamCreateSendsConfiguredTeamID(t *testing.T) { + var captured map[string]interface{} + srv := newTeamTestServer(t, &captured, `{"team_id":"platform-team","team_info":{"team_id":"platform-team","team_alias":"platform"},"keys":[],"team_memberships":[]}`) + defer srv.Close() + + d := newTeamResourceData(t, map[string]interface{}{ + "team_id": "platform-team", + "team_alias": "platform", + }) + + if err := resourceLiteLLMTeamCreate(d, NewClient(srv.URL, "test-key", true)); err != nil { + t.Fatalf("create failed: %v", err) + } + + if got := captured["team_id"]; got != "platform-team" { + t.Fatalf("payload team_id = %v, want platform-team", got) + } + if got := d.Id(); got != "platform-team" { + t.Fatalf("resource id = %q, want platform-team", got) + } + if got := d.Get("team_id"); got != "platform-team" { + t.Fatalf("state team_id = %v, want platform-team", got) + } +} + +func TestTeamCreateGeneratesTeamIDWhenUnset(t *testing.T) { + var captured map[string]interface{} + srv := newTeamTestServer(t, &captured, `{"team_id":"x","team_info":{"team_alias":"eng"},"keys":[],"team_memberships":[]}`) + defer srv.Close() + + d := newTeamResourceData(t, map[string]interface{}{"team_alias": "eng"}) + + if err := resourceLiteLLMTeamCreate(d, NewClient(srv.URL, "test-key", true)); err != nil { + t.Fatalf("create failed: %v", err) + } + + sent, _ := captured["team_id"].(string) + if _, err := uuid.Parse(sent); err != nil { + t.Fatalf("payload team_id = %q, want a generated UUID: %v", sent, err) + } + if d.Id() != sent || d.Get("team_id") != sent { + t.Fatalf("id = %q, state team_id = %v, want both to equal the sent id %q", d.Id(), d.Get("team_id"), sent) + } +} + +func TestTeamReadSetsTeamIDFromResourceID(t *testing.T) { + var captured map[string]interface{} + srv := newTeamTestServer(t, &captured, `{"team_id":"imported-team","team_info":{"team_id":"imported-team","team_alias":"imported"},"keys":[],"team_memberships":[]}`) + defer srv.Close() + + d := newTeamResourceData(t, map[string]interface{}{}) + d.SetId("imported-team") + + if err := resourceLiteLLMTeamRead(d, NewClient(srv.URL, "test-key", true)); err != nil { + t.Fatalf("read failed: %v", err) + } + if got := d.Get("team_id"); got != "imported-team" { + t.Fatalf("team_id = %v, want imported-team", got) + } +} + +func TestTeamIDChangeForcesReplacement(t *testing.T) { + res := ResourceLiteLLMTeam() + priorData := schema.TestResourceDataRaw(t, res.Schema, map[string]interface{}{ + "team_id": "old-team", + "team_alias": "eng", + }) + priorData.SetId("old-team") + config := terraform.NewResourceConfigRaw(map[string]interface{}{ + "team_id": "new-team", + "team_alias": "eng", + }) + diff, err := res.Diff(context.Background(), priorData.State(), config, nil) + if err != nil { + t.Fatalf("diff failed: %v", err) + } + if diff == nil || !diff.RequiresNew() { + t.Fatalf("changing team_id must force replacement, diff = %+v", diff) + } +} + func TestTeamReadMapsTeamInfoEnvelope(t *testing.T) { var captured map[string]interface{} srv := newTeamTestServer(t, &captured, teamInfoWithSoftBudget) From cb9c3a913708845254dd289f8ffa0fb25ec4c1d9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:46:26 -0700 Subject: [PATCH 287/411] fix(terraform): send litellm_key model_max_budget as per-model BudgetConfig objects (#40450) * fix(terraform): send litellm_key model_max_budget as per-model BudgetConfig objects The key resource typed model_max_budget as map(number) and forwarded the bare numbers to /key/generate, which the proxy rejects with a 500 ('int' object is not iterable) because each model entry must be a BudgetConfig object. The attribute is now a JSON string of per-model budget objects, matching litellm_user, litellm_budget and litellm_tag, with a schema version 1 state upgrader that drops the old map value so existing state keeps loading Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(terraform): reject litellm_key model_max_budget JSON that is not per-model budget objects Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(terraform): reject unknown or empty per-model budget fields in litellm_key model_max_budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/CHANGELOG.md | 1 + terraform/provider/README.md | 9 +- terraform/provider/docs/resources/key.md | 11 ++- terraform/provider/litellm/resource_key.go | 91 ++++++++++++++++-- .../provider/litellm/resource_key_test.go | 96 +++++++++++++++++++ .../provider/litellm/resource_key_utils.go | 4 +- 6 files changed, 195 insertions(+), 17 deletions(-) diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index de54207ea5a..798c967c736 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -45,6 +45,7 @@ longer signal it. ### Changed +- **key** (breaking): `model_max_budget` on `litellm_key` is now a JSON string of per-model budget objects (`jsonencode({"gpt-4o-mini" = {budget_limit = 50, time_period = "30d"}})`), matching `litellm_user`, `litellm_budget` and `litellm_tag`. The old `map(number)` form sent bare numbers to `/key/generate`, which the proxy rejects with a 500 (`'int' object is not iterable`), so every key with a non-empty `model_max_budget` failed to apply. Existing state upgrades automatically (schema version 1) and the attribute is refilled from the proxy on the next read; configurations still using the map form must be rewritten - **Versioning**: the provider is now published at the LiteLLM version, from the same commit as the proxy, on every LiteLLM release (dev, rc, stable). The `0.x` line ends at `0.4.0`; a `~> 0.4` constraint will not receive further releases, so re-pin to the LiteLLM version your proxy runs (for example `~> 1.99.0`). Existing `0.x` versions remain in the registry and keep verifying ## [0.4.0] - 2026-08-06 diff --git a/terraform/provider/README.md b/terraform/provider/README.md index 0a6d15c7844..b392fd6279d 100644 --- a/terraform/provider/README.md +++ b/terraform/provider/README.md @@ -103,9 +103,12 @@ resource "litellm_key" "example_key" { permissions = { can_create_keys = "true" } - model_max_budget = { - "gpt-4" = 50.0 - } + model_max_budget = jsonencode({ + "gpt-4" = { + budget_limit = 50.0 + time_period = "30d" + } + }) model_rpm_limit = { "claude-3.5-sonnet" = 30 } diff --git a/terraform/provider/docs/resources/key.md b/terraform/provider/docs/resources/key.md index 5094b77cbec..8b228914649 100644 --- a/terraform/provider/docs/resources/key.md +++ b/terraform/provider/docs/resources/key.md @@ -30,9 +30,12 @@ resource "litellm_key" "example" { permissions = { "can_create_keys" = "true" } - model_max_budget = { - "gpt-4" = 50.0 - } + model_max_budget = jsonencode({ + "gpt-4" = { + budget_limit = 50.0 + time_period = "30d" + } + }) model_rpm_limit = { "gpt-3.5-turbo" = 30 } @@ -81,7 +84,7 @@ The following arguments are supported: * `permissions` - (Optional) Permissions associated with this key. This defines what actions are allowed with this key. -* `model_max_budget` - (Optional) Maximum budget per model. This allows setting different budget limits for each model. +* `model_max_budget` - (Optional) JSON string of per-model budget config, e.g. `jsonencode({"gpt-4" = {budget_limit = 50.0, time_period = "30d"}})`. Each model maps to an object with `budget_limit` (or `max_budget`), `time_period` (or `budget_duration`), `tpm_limit` and `rpm_limit`. * `model_rpm_limit` - (Optional) Requests per minute limit per model. This allows setting different RPM limits for each model. diff --git a/terraform/provider/litellm/resource_key.go b/terraform/provider/litellm/resource_key.go index a37041fe7e8..5d8bc5667a9 100644 --- a/terraform/provider/litellm/resource_key.go +++ b/terraform/provider/litellm/resource_key.go @@ -2,6 +2,7 @@ package litellm import ( "context" + "encoding/json" "fmt" "github.com/hashicorp/go-cty/cty" @@ -10,7 +11,7 @@ import ( ) func resourceKey() *schema.Resource { - return &schema.Resource{ + r := &schema.Resource{ CreateContext: resourceKeyCreate, ReadContext: resourceKeyRead, UpdateContext: resourceKeyUpdate, @@ -18,6 +19,7 @@ func resourceKey() *schema.Resource { Importer: &schema.ResourceImporter{ StateContext: schema.ImportStatePassthroughContext, }, + SchemaVersion: 1, Schema: map[string]*schema.Schema{ "key": { Type: schema.TypeString, @@ -105,9 +107,11 @@ func resourceKey() *schema.Resource { Elem: &schema.Schema{Type: schema.TypeString}, }, "model_max_budget": { - Type: schema.TypeMap, - Optional: true, - Elem: &schema.Schema{Type: schema.TypeFloat, Computed: true}, + Type: schema.TypeString, + Optional: true, + ValidateFunc: validateKeyModelMaxBudget, + DiffSuppressFunc: budgetSuppressEquivalentJSON, + Description: "JSON string of per-model budget config (e.g. '{\"gpt-4o-mini\": {\"budget_limit\": 50, \"time_period\": \"30d\"}}')", }, "model_rpm_limit": { Type: schema.TypeMap, @@ -182,6 +186,79 @@ func resourceKey() *schema.Resource { }, }, } + r.StateUpgraders = []schema.StateUpgrader{{ + Version: 0, + Type: resourceKeyV0Type(r.Schema), + Upgrade: resourceKeyStateUpgradeV0, + }} + return r +} + +// Schema version 0 typed model_max_budget as map(number), which the proxy +// rejects; version 1 stores the per-model BudgetConfig objects as a JSON string. +func resourceKeyV0Type(current map[string]*schema.Schema) cty.Type { + v0 := make(map[string]*schema.Schema, len(current)) + for k, v := range current { + v0[k] = v + } + v0["model_max_budget"] = &schema.Schema{ + Type: schema.TypeMap, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeFloat}, + } + return (&schema.Resource{Schema: v0}).CoreConfigSchema().ImpliedType() +} + +func resourceKeyStateUpgradeV0(_ context.Context, rawState map[string]interface{}, _ interface{}) (map[string]interface{}, error) { + delete(rawState, "model_max_budget") + return rawState, nil +} + +var keyModelBudgetFields = map[string]bool{ + "budget_limit": true, + "max_budget": true, + "time_period": true, + "budget_duration": true, + "tpm_limit": true, + "rpm_limit": true, +} + +func validateKeyModelMaxBudget(v interface{}, k string) ([]string, []error) { + var parsed map[string]json.RawMessage + if err := json.Unmarshal([]byte(v.(string)), &parsed); err != nil || parsed == nil { + return nil, []error{fmt.Errorf("%q must be a JSON object keyed by model name, got %s", k, v)} + } + for model, cfg := range parsed { + var budget map[string]json.RawMessage + if err := json.Unmarshal(cfg, &budget); err != nil || len(budget) == 0 { + return nil, []error{fmt.Errorf("%q[%q] must be a budget object such as {\"budget_limit\": 50, \"time_period\": \"30d\"}, got %s", k, model, cfg)} + } + for field := range budget { + if !keyModelBudgetFields[field] { + return nil, []error{fmt.Errorf("%q[%q] has unknown budget field %q; supported fields are budget_limit, max_budget, time_period, budget_duration, tpm_limit, rpm_limit", k, model, field)} + } + } + } + return nil, nil +} + +func parseKeyModelMaxBudget(raw string) map[string]interface{} { + var parsed map[string]interface{} + if err := json.Unmarshal([]byte(raw), &parsed); err != nil || parsed == nil { + return map[string]interface{}{} + } + return parsed +} + +func keyModelMaxBudgetJSON(modelMaxBudget map[string]interface{}) string { + if len(modelMaxBudget) == 0 { + return "" + } + encoded, err := json.Marshal(modelMaxBudget) + if err != nil { + return "" + } + return string(encoded) } func resourceKeyCreate(ctx context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics { @@ -342,7 +419,7 @@ func mapResourceDataToKey(d *schema.ResourceData, key *Key) { key.Aliases = d.Get("aliases").(map[string]interface{}) key.Config = d.Get("config").(map[string]interface{}) key.Permissions = d.Get("permissions").(map[string]interface{}) - key.ModelMaxBudget = d.Get("model_max_budget").(map[string]interface{}) + key.ModelMaxBudget = parseKeyModelMaxBudget(d.Get("model_max_budget").(string)) key.ModelRPMLimit = d.Get("model_rpm_limit").(map[string]interface{}) key.ModelTPMLimit = d.Get("model_tpm_limit").(map[string]interface{}) key.Guardrails = expandStringList(d.Get("guardrails").([]interface{})) @@ -415,9 +492,7 @@ func mapKeyToResourceData(d *schema.ResourceData, key *Key) { if key.Permissions != nil { d.Set("permissions", key.Permissions) } - if key.ModelMaxBudget != nil { - d.Set("model_max_budget", key.ModelMaxBudget) - } + d.Set("model_max_budget", keyModelMaxBudgetJSON(key.ModelMaxBudget)) if key.ModelRPMLimit != nil { d.Set("model_rpm_limit", key.ModelRPMLimit) } diff --git a/terraform/provider/litellm/resource_key_test.go b/terraform/provider/litellm/resource_key_test.go index d8f3cfe86b3..30108856a3a 100644 --- a/terraform/provider/litellm/resource_key_test.go +++ b/terraform/provider/litellm/resource_key_test.go @@ -195,6 +195,102 @@ func TestCreateKeySendsConfigSuppliedKey(t *testing.T) { } } +// The proxy validates each model_max_budget entry as a BudgetConfig object and +// 500s on a bare number, so the JSON string must reach /key/generate as nested +// objects and the proxy's response must map back to equivalent JSON in state. +func TestCreateKeySendsModelMaxBudgetAsBudgetObjects(t *testing.T) { + var captured map[string]interface{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/key/generate" { + body, _ := io.ReadAll(r.Body) + json.Unmarshal(body, &captured) + w.Write([]byte(`{"key": "sk-test", "token_id": "hash-1"}`)) + return + } + w.Write([]byte(`{"key": "hash-1", "info": {"model_max_budget": {"gpt-4o-mini": {"budget_limit": 50, "time_period": "30d", "rpm_limit": 60}}}}`)) + })) + defer srv.Close() + + client := NewClient(srv.URL, "test-key", true) + d := newKeyResourceData(t, map[string]interface{}{ + "model_max_budget": `{"gpt-4o-mini": {"budget_limit": 50, "time_period": "30d"}}`, + }) + + if diags := resourceKeyCreate(context.Background(), d, client); diags.HasError() { + t.Fatalf("create returned error: %v", diags) + } + + budgets, ok := captured["model_max_budget"].(map[string]interface{}) + if !ok { + t.Fatalf("create payload model_max_budget = %v, want object", captured["model_max_budget"]) + } + cfg, ok := budgets["gpt-4o-mini"].(map[string]interface{}) + if !ok { + t.Fatalf("model_max_budget[gpt-4o-mini] = %v, want BudgetConfig object", budgets["gpt-4o-mini"]) + } + if cfg["budget_limit"] != float64(50) || cfg["time_period"] != "30d" { + t.Errorf("BudgetConfig = %v, want budget_limit 50 and time_period 30d", cfg) + } + + var state map[string]interface{} + if err := json.Unmarshal([]byte(d.Get("model_max_budget").(string)), &state); err != nil { + t.Fatalf("state model_max_budget %q is not JSON: %v", d.Get("model_max_budget"), err) + } + if got, _ := state["gpt-4o-mini"].(map[string]interface{}); got["budget_limit"] != float64(50) || got["rpm_limit"] != float64(60) { + t.Errorf("state model_max_budget = %v, want the BudgetConfig read back from /key/info", state) + } +} + +// Schema version 0 stored model_max_budget as map(number); that state cannot +// decode into the version 1 string attribute, so the upgrader must drop it. +func TestKeyStateUpgradeV0DropsMapModelMaxBudget(t *testing.T) { + upgraded, err := resourceKey().StateUpgraders[0].Upgrade(context.Background(), map[string]interface{}{ + "id": "hash-1", + "key_alias": "legacy", + "model_max_budget": map[string]interface{}{"gpt-4o-mini": 50.0}, + }, nil) + if err != nil { + t.Fatalf("upgrade returned error: %v", err) + } + if _, present := upgraded["model_max_budget"]; present { + t.Errorf("upgraded state still carries map model_max_budget: %v", upgraded["model_max_budget"]) + } + if upgraded["key_alias"] != "legacy" { + t.Errorf("upgrade dropped unrelated attribute: %v", upgraded) + } +} + +func TestKeyModelMaxBudgetValidationRequiresBudgetObjects(t *testing.T) { + validate := resourceKey().Schema["model_max_budget"].ValidateFunc + for _, valid := range []string{ + `{}`, + `{"gpt-4o-mini": {"budget_limit": 50, "time_period": "30d"}}`, + `{"gpt-4o-mini": {"max_budget": 50, "rpm_limit": 60}, "gpt-4o": {"budget_duration": "1d", "tpm_limit": 1000}}`, + } { + if _, errs := validate(valid, "model_max_budget"); len(errs) != 0 { + t.Errorf("validate(%s) = %v, want accepted", valid, errs) + } + } + for _, invalid := range []string{ + `null`, + `[]`, + `"gpt-4o-mini"`, + `50`, + `{"gpt-4o-mini": 50}`, + `{"gpt-4o-mini": null}`, + `{"gpt-4o-mini": [50]}`, + `{"gpt-4o-mini": {}}`, + `{"gpt-4o-mini": {"budget_limt": 50}}`, + `{"gpt-4o-mini": {"budget_limit": 50, "max_tokens": 100}}`, + `not json`, + } { + if _, errs := validate(invalid, "model_max_budget"); len(errs) == 0 { + t.Errorf("validate(%s) accepted a value that would send no per-model budget", invalid) + } + } +} + // The proxy 400s on budget_duration: "", so an unset duration must be // omitted from the update payload entirely. func TestUpdateKeyOmitsEmptyBudgetDuration(t *testing.T) { diff --git a/terraform/provider/litellm/resource_key_utils.go b/terraform/provider/litellm/resource_key_utils.go index d426fec05b2..7046ef7097a 100644 --- a/terraform/provider/litellm/resource_key_utils.go +++ b/terraform/provider/litellm/resource_key_utils.go @@ -65,7 +65,7 @@ func buildKeyData(d *schema.ResourceData) map[string]interface{} { keyData["permissions"] = v.(map[string]interface{}) } if v, ok := d.GetOkExists("model_max_budget"); ok { - keyData["model_max_budget"] = v.(map[string]interface{}) + keyData["model_max_budget"] = parseKeyModelMaxBudget(v.(string)) } if v, ok := d.GetOkExists("model_rpm_limit"); ok { keyData["model_rpm_limit"] = v.(map[string]interface{}) @@ -107,7 +107,7 @@ func setKeyResourceData(d *schema.ResourceData, key *Key) error { "aliases": key.Aliases, "config": key.Config, "permissions": key.Permissions, - "model_max_budget": key.ModelMaxBudget, + "model_max_budget": keyModelMaxBudgetJSON(key.ModelMaxBudget), "model_rpm_limit": key.ModelRPMLimit, "model_tpm_limit": key.ModelTPMLimit, "guardrails": key.Guardrails, From fb603fdb6b2894f4bd21f33293c025c1f73e4dc6 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:48:35 -0700 Subject: [PATCH 288/411] fix(terraform): keep prior state when a virtual key update is rejected (#40512) resourceKeyUpdate now calls d.Partial(true) on a failed /key/update so the SDK does not persist the rejected planned values into state, which made the next plan report no changes and silently dropped the update. Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/litellm/resource_key.go | 1 + .../provider/litellm/resource_key_test.go | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/terraform/provider/litellm/resource_key.go b/terraform/provider/litellm/resource_key.go index 5d8bc5667a9..3b14f7ca655 100644 --- a/terraform/provider/litellm/resource_key.go +++ b/terraform/provider/litellm/resource_key.go @@ -315,6 +315,7 @@ func resourceKeyUpdate(ctx context.Context, d *schema.ResourceData, m interface{ metadata, err := plannedKeyMetadata(c, d) if err != nil { + d.Partial(true) return diag.FromErr(fmt.Errorf("error updating key: %s", err)) } key.Metadata = metadata diff --git a/terraform/provider/litellm/resource_key_test.go b/terraform/provider/litellm/resource_key_test.go index 30108856a3a..5e6385735c6 100644 --- a/terraform/provider/litellm/resource_key_test.go +++ b/terraform/provider/litellm/resource_key_test.go @@ -319,6 +319,47 @@ func TestUpdateKeyOmitsEmptyBudgetDuration(t *testing.T) { } } +func TestResourceKeyUpdateFailureKeepsPriorState(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/key/update" { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"error":{"message":"Invalid budget_duration 'bad'"}}`)) + return + } + w.Write([]byte(`{"key":"hash-1","info":{"key_alias":"demo","models":["fake-model"]}}`)) + })) + defer srv.Close() + + res := resourceKey() + priorData := newKeyResourceData(t, map[string]interface{}{ + "key_alias": "demo", + "models": []interface{}{"fake-model"}, + }) + priorData.SetId("hash-1") + prior := priorData.State() + config := terraform.NewResourceConfigRaw(map[string]interface{}{ + "key_alias": "demo", + "models": []interface{}{"fake-model"}, + "budget_duration": "bad", + }) + diff, err := res.Diff(context.Background(), prior, config, nil) + if err != nil { + t.Fatalf("diff failed: %v", err) + } + + newState, diags := res.Apply(context.Background(), prior, diff, NewClient(srv.URL, "test-key", true)) + if !diags.HasError() { + t.Fatal("apply succeeded, want the proxy's 400 surfaced as an error") + } + if got, ok := newState.Attributes["budget_duration"]; ok { + t.Errorf("failed update persisted budget_duration=%q into state, want it absent", got) + } + if newState.Attributes["key_alias"] != "demo" { + t.Errorf("prior key_alias lost from state: %v", newState.Attributes) + } +} + // /key/info nests the key's fields under "info"; GetKey must unwrap that // envelope or reads map nothing back into state. func TestGetKeyUnwrapsInfoEnvelope(t *testing.T) { From ea106fd8be7666111623c0ea7480aa4e1634cc82 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 02:57:07 +0000 Subject: [PATCH 289/411] fix(terraform): send changed litellm_key duration on /key/update so expires is recomputed (#40511) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/docs/resources/key.md | 2 +- terraform/provider/litellm/client.go | 3 ++ terraform/provider/litellm/resource_key.go | 8 +++-- .../provider/litellm/resource_key_test.go | 35 +++++++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/terraform/provider/docs/resources/key.md b/terraform/provider/docs/resources/key.md index 8b228914649..0ef0688830f 100644 --- a/terraform/provider/docs/resources/key.md +++ b/terraform/provider/docs/resources/key.md @@ -76,7 +76,7 @@ The following arguments are supported: * `key_alias` - (Optional) Alias for this key. This provides a human-readable identifier for the key. -* `duration` - (Optional) Duration for which this key is valid. This sets an expiration time for the key. +* `duration` - (Optional) How long the key stays valid, e.g. "30d" or "12h". The proxy stores this as an absolute `expires` timestamp. Changing the value resets the expiry to the time of the update plus the new duration; removing it from the configuration leaves the current expiry in place. * `aliases` - (Optional) Map of model aliases. This allows you to create custom names for models when using this key. diff --git a/terraform/provider/litellm/client.go b/terraform/provider/litellm/client.go index 0dfbe3dd3c9..58e52aabefc 100644 --- a/terraform/provider/litellm/client.go +++ b/terraform/provider/litellm/client.go @@ -103,6 +103,9 @@ func (c *Client) UpdateKey(key *Key) (*Key, error) { if key.BudgetDuration != "" { updateData["budget_duration"] = key.BudgetDuration } + if key.Duration != "" { + updateData["duration"] = key.Duration + } // Only add pointer fields if they are explicitly set if key.MaxBudget != nil { diff --git a/terraform/provider/litellm/resource_key.go b/terraform/provider/litellm/resource_key.go index 3b14f7ca655..95bf88c0145 100644 --- a/terraform/provider/litellm/resource_key.go +++ b/terraform/provider/litellm/resource_key.go @@ -88,8 +88,9 @@ func resourceKey() *schema.Resource { Optional: true, }, "duration": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + Description: "How long the key stays valid, e.g. \"30d\" or \"12h\". Changing it resets the expiry to the time of the update plus the new duration; removing it leaves the current expiry in place", }, "aliases": { Type: schema.TypeMap, @@ -310,6 +311,9 @@ func resourceKeyUpdate(ctx context.Context, d *schema.ResourceData, m interface{ key := &Key{Key: d.Id()} mapResourceDataToKey(d, key) + if !d.HasChange("duration") { + key.Duration = "" + } key.ModelRPMLimit = changedMap(d, "model_rpm_limit") key.ModelTPMLimit = changedMap(d, "model_tpm_limit") diff --git a/terraform/provider/litellm/resource_key_test.go b/terraform/provider/litellm/resource_key_test.go index 5e6385735c6..722a90ee02d 100644 --- a/terraform/provider/litellm/resource_key_test.go +++ b/terraform/provider/litellm/resource_key_test.go @@ -524,3 +524,38 @@ func TestKeyReadKeepsOnlyDeclaredMetadata(t *testing.T) { t.Errorf("metadata in state = %v, want %v", got, want) } } + +func TestKeyUpdateSendsChangedDuration(t *testing.T) { + proxy := &fakeKeyProxy{metadata: map[string]interface{}{}} + srv := httptest.NewServer(proxy.handler()) + defer srv.Close() + client := NewClient(srv.URL, "test-key", true) + + applyKeyUpdate(t, client, + map[string]string{"key_alias": "alias-1", "duration": "30d"}, + map[string]interface{}{"key_alias": "alias-1", "duration": "90d"}, + ) + + if got := proxy.updates[0]["duration"]; got != "90d" { + t.Errorf("update payload duration = %v, want 90d", got) + } +} + +func TestKeyUpdateOmitsUnchangedDuration(t *testing.T) { + proxy := &fakeKeyProxy{metadata: map[string]interface{}{}} + srv := httptest.NewServer(proxy.handler()) + defer srv.Close() + client := NewClient(srv.URL, "test-key", true) + + applyKeyUpdate(t, client, + map[string]string{"key_alias": "alias-1", "duration": "30d"}, + map[string]interface{}{"key_alias": "alias-2", "duration": "30d"}, + ) + + if got := proxy.updates[0]["key_alias"]; got != "alias-2" { + t.Fatalf("update payload key_alias = %v, want alias-2", got) + } + if v, present := proxy.updates[0]["duration"]; present { + t.Errorf("update payload unexpectedly contains duration = %v", v) + } +} From 5264f8ed48de6615c903e459aced1dd04137b1b9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 02:57:51 +0000 Subject: [PATCH 290/411] fix(terraform): drop litellm_key from state on 404 instead of failing the plan (#40443) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/litellm/client.go | 20 +++++++++- terraform/provider/litellm/resource_key.go | 2 + .../provider/litellm/resource_key_test.go | 39 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/terraform/provider/litellm/client.go b/terraform/provider/litellm/client.go index 58e52aabefc..76526a8595c 100644 --- a/terraform/provider/litellm/client.go +++ b/terraform/provider/litellm/client.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "log" @@ -19,6 +20,20 @@ type Client struct { InsecureSkipVerify bool } +type apiError struct { + StatusCode int + Body string +} + +func (e *apiError) Error() string { + return fmt.Sprintf("API request failed with status code %d: %s", e.StatusCode, e.Body) +} + +func isNotFound(err error) bool { + var apiErr *apiError + return errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound +} + func NewClient(apiBase, apiKey string, insecureSkipVerify bool) *Client { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}, @@ -57,6 +72,9 @@ func (c *Client) CreateKey(key *Key) (*Key, error) { func (c *Client) GetKey(keyID string) (*Key, error) { resp, err := c.sendRequest("GET", fmt.Sprintf("/key/info?key=%s", keyID), nil) + if isNotFound(err) { + return nil, nil + } if err != nil { return nil, err } @@ -377,7 +395,7 @@ func (c *Client) sendRequest(method, path string, body interface{}) (map[string] log.Printf("Response body: %s", c.redactSensitiveData(string(bodyBytes))) if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("API request failed with status code %d: %s", resp.StatusCode, string(bodyBytes)) + return nil, &apiError{StatusCode: resp.StatusCode, Body: string(bodyBytes)} } var result map[string]interface{} diff --git a/terraform/provider/litellm/resource_key.go b/terraform/provider/litellm/resource_key.go index 95bf88c0145..ffdc448f5fe 100644 --- a/terraform/provider/litellm/resource_key.go +++ b/terraform/provider/litellm/resource_key.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log" "github.com/hashicorp/go-cty/cty" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" @@ -297,6 +298,7 @@ func resourceKeyRead(ctx context.Context, d *schema.ResourceData, m interface{}) } if key == nil { + log.Printf("[WARN] Key %s not found, removing from state", d.Id()) d.SetId("") return nil } diff --git a/terraform/provider/litellm/resource_key_test.go b/terraform/provider/litellm/resource_key_test.go index 722a90ee02d..1da8528576d 100644 --- a/terraform/provider/litellm/resource_key_test.go +++ b/terraform/provider/litellm/resource_key_test.go @@ -394,6 +394,45 @@ func TestGetKeyUnwrapsInfoEnvelope(t *testing.T) { } } +func TestResourceKeyReadDropsMissingKeyFromState(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + w.Write([]byte(`{"error":{"message":"Key not found in database","type":"not_found_error","param":"key","code":"404"}}`)) + })) + defer srv.Close() + + d := newKeyResourceData(t, map[string]interface{}{"key_alias": "stale"}) + d.SetId("deleted-out-of-band") + + diags := resourceKeyRead(context.Background(), d, NewClient(srv.URL, "test-key", true)) + if diags.HasError() { + t.Fatalf("read of a missing key must not error, got: %v", diags) + } + if d.Id() != "" { + t.Errorf("Id = %q, want empty so Terraform plans a recreate", d.Id()) + } +} + +func TestResourceKeyReadStillFailsOnNon404Errors(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{"error":{"message":"db down"}}`)) + })) + defer srv.Close() + + d := newKeyResourceData(t, map[string]interface{}{"key_alias": "live"}) + d.SetId("still-exists") + + diags := resourceKeyRead(context.Background(), d, NewClient(srv.URL, "test-key", true)) + if !diags.HasError() { + t.Fatal("a 500 from /key/info must surface as an error, not be treated as a deleted key") + } + if d.Id() != "still-exists" { + t.Errorf("Id = %q, want unchanged on a transient error", d.Id()) + } +} + // fakeKeyProxy serves /key/info from stored metadata and applies /key/update // the way the proxy does: an absent "metadata" keeps the stored map, a // present one replaces it wholesale. From 87e2026f0a440279c3a1b32d7351df91d13d2d29 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:10:42 +0000 Subject: [PATCH 291/411] fix(terraform): read key fields the proxy stores in metadata back from /key/info (#40513) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/CHANGELOG.md | 1 + terraform/provider/litellm/client.go | 28 ++++++ .../provider/litellm/resource_key_test.go | 88 +++++++++++++++++++ 3 files changed, 117 insertions(+) diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index 798c967c736..e5e0a164a83 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -39,6 +39,7 @@ longer signal it. - **team**: Read now decodes the `team_info` envelope `/team/info` actually returns, so team attributes refresh from the proxy instead of always falling back to the prior state - **key**: Read now unwraps the `info` envelope `/key/info` actually returns; previously reads mapped nothing back into state, so drift on a key was never detected +- **key**: Read now picks up `model_rpm_limit`, `model_tpm_limit`, `guardrails`, `tags`, `enforced_params`, `allowed_passthrough_routes`, `rpm_limit_type`, `tpm_limit_type` and `prompts` from `info.metadata`, where the proxy actually stores them; previously they stayed empty in state, so a matching config showed a permanent phantom diff on them and out-of-band changes to them were never detected - **key**: Updates no longer send an empty `budget_duration`, which the proxy rejects with a 400; any update to a key without a configured `budget_duration` previously failed outright - **key**: A config-supplied `key` value (write-only) is now forwarded to `/key/generate`; previously it was silently dropped and the proxy generated a random key instead - **security**: The `litellm_key` data source and `litellm_key_block` resource normalize raw `sk-` keys to their SHA-256 token hash before building request URLs and resource IDs, so plaintext keys no longer land in reverse-proxy access logs, Terraform plan output, or state IDs diff --git a/terraform/provider/litellm/client.go b/terraform/provider/litellm/client.go index 76526a8595c..e68b8a3a80b 100644 --- a/terraform/provider/litellm/client.go +++ b/terraform/provider/litellm/client.go @@ -87,12 +87,40 @@ func (c *Client) GetKey(keyID string) (*Key, error) { info["key"] = k } } + hoistKeyFieldsStoredInMetadata(info) return c.parseKeyResponse(info) } return c.parseKeyResponse(resp) } +var keyFieldsStoredInMetadata = []string{ + "model_rpm_limit", + "model_tpm_limit", + "guardrails", + "tags", + "enforced_params", + "allowed_passthrough_routes", + "rpm_limit_type", + "tpm_limit_type", + "prompts", +} + +func hoistKeyFieldsStoredInMetadata(info map[string]interface{}) { + metadata, ok := info["metadata"].(map[string]interface{}) + if !ok { + return + } + for _, field := range keyFieldsStoredInMetadata { + if existing, present := info[field]; present && existing != nil { + continue + } + if v, present := metadata[field]; present { + info[field] = v + } + } +} + func (c *Client) UpdateKey(key *Key) (*Key, error) { // Create a new map with only the fields that can be updated updateData := map[string]interface{}{ diff --git a/terraform/provider/litellm/resource_key_test.go b/terraform/provider/litellm/resource_key_test.go index 1da8528576d..66291eadcc5 100644 --- a/terraform/provider/litellm/resource_key_test.go +++ b/terraform/provider/litellm/resource_key_test.go @@ -394,6 +394,94 @@ func TestGetKeyUnwrapsInfoEnvelope(t *testing.T) { } } +func TestGetKeyReadsFieldsStoredInMetadata(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{ + "key": "hash-1", + "info": { + "models": ["gpt-4o-mini"], + "metadata": { + "team": "core-infra", + "model_rpm_limit": {"gpt-4o-mini": 7}, + "model_tpm_limit": {"gpt-4o-mini": 10000}, + "guardrails": ["pii-guard"], + "tags": ["prod"], + "enforced_params": ["user"], + "allowed_passthrough_routes": ["/v1/foo"], + "rpm_limit_type": "guaranteed_throughput", + "tpm_limit_type": "dynamic", + "prompts": ["p1"] + } + } + }`)) + })) + defer srv.Close() + + client := NewClient(srv.URL, "test-key", true) + key, err := client.GetKey("hash-1") + if err != nil { + t.Fatalf("GetKey returned error: %v", err) + } + if got, ok := key.ModelRPMLimit["gpt-4o-mini"].(float64); !ok || got != 7 { + t.Errorf("ModelRPMLimit = %v, want gpt-4o-mini=7 read from metadata", key.ModelRPMLimit) + } + if got, ok := key.ModelTPMLimit["gpt-4o-mini"].(float64); !ok || got != 10000 { + t.Errorf("ModelTPMLimit = %v, want gpt-4o-mini=10000 read from metadata", key.ModelTPMLimit) + } + if len(key.Guardrails) != 1 || key.Guardrails[0] != "pii-guard" { + t.Errorf("Guardrails = %v, want [pii-guard]", key.Guardrails) + } + if len(key.Tags) != 1 || key.Tags[0] != "prod" { + t.Errorf("Tags = %v, want [prod]", key.Tags) + } + if len(key.EnforcedParams) != 1 || key.EnforcedParams[0] != "user" { + t.Errorf("EnforcedParams = %v, want [user]", key.EnforcedParams) + } + if len(key.AllowedPassthroughRoutes) != 1 || key.AllowedPassthroughRoutes[0] != "/v1/foo" { + t.Errorf("AllowedPassthroughRoutes = %v, want [/v1/foo]", key.AllowedPassthroughRoutes) + } + if key.RPMLimitType != "guaranteed_throughput" || key.TPMLimitType != "dynamic" { + t.Errorf("limit types = %q/%q, want guaranteed_throughput/dynamic", key.RPMLimitType, key.TPMLimitType) + } + if len(key.Prompts) != 1 || key.Prompts[0] != "p1" { + t.Errorf("Prompts = %v, want [p1]", key.Prompts) + } + if key.Metadata["team"] != "core-infra" { + t.Errorf("Metadata = %v, want team=core-infra preserved", key.Metadata) + } +} + +func TestGetKeyPrefersTopLevelOverMetadataCopy(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{ + "key": "hash-1", + "info": { + "tags": ["top-level"], + "guardrails": null, + "metadata": { + "tags": ["from-metadata"], + "guardrails": ["from-metadata"] + } + } + }`)) + })) + defer srv.Close() + + client := NewClient(srv.URL, "test-key", true) + key, err := client.GetKey("hash-1") + if err != nil { + t.Fatalf("GetKey returned error: %v", err) + } + if len(key.Tags) != 1 || key.Tags[0] != "top-level" { + t.Errorf("Tags = %v, want [top-level]", key.Tags) + } + if len(key.Guardrails) != 1 || key.Guardrails[0] != "from-metadata" { + t.Errorf("Guardrails = %v, want [from-metadata] (null top-level must not shadow)", key.Guardrails) + } +} + func TestResourceKeyReadDropsMissingKeyFromState(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") From 935611d25d092c5623b44e08d89d7c2e3d565933 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:20:07 -0700 Subject: [PATCH 292/411] fix(mcp): respect optional discovery capabilities and quiet unsupported methods --- litellm/experimental_mcp_client/client.py | 53 ++++- .../test_mcp_client.py | 209 ++++++++++++++++++ 2 files changed, 254 insertions(+), 8 deletions(-) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 8c7f4557992..421a3c84d75 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -54,18 +54,22 @@ def missing_streamable_http_client_error() -> ImportError: ) -from mcp.types import CallToolRequestParams as MCPCallToolRequestParams -from mcp.types import CallToolResult as MCPCallToolResult from mcp.types import ( + METHOD_NOT_FOUND, ClientResult, GetPromptRequestParams, GetPromptResult, + ListPromptsResult, + ListResourcesResult, + ListResourceTemplatesResult, Prompt, ResourceTemplate, ServerNotification, ServerRequest, TextContent, ) +from mcp.types import CallToolRequestParams as MCPCallToolRequestParams +from mcp.types import CallToolResult as MCPCallToolResult from mcp.types import Tool as MCPTool from pydantic import AnyUrl @@ -777,8 +781,19 @@ class MCPClient: """List available prompts from the server.""" verbose_logger.debug("MCP client listing tools from %s", self.server_url or "stdio") - async def _list_prompts_operation(session: ClientSession): - return await session.list_prompts() + async def _list_prompts_operation(session: ClientSession) -> ListPromptsResult: + capabilities: Final = session.get_server_capabilities() + if capabilities is not None and capabilities.prompts is None: + return ListPromptsResult(prompts=[]) + try: + return await session.list_prompts() + except McpError as error: + if error.error.code != METHOD_NOT_FOUND: + raise + verbose_logger.debug( + "MCP client list_prompts is unsupported by %s: %s", self.server_url or "stdio", error + ) + return ListPromptsResult(prompts=[]) try: result: Final = await self.run_with_session(_list_prompts_operation) @@ -854,8 +869,19 @@ class MCPClient: """List available resources from the server.""" verbose_logger.debug("MCP client listing resources from %s", self.server_url or "stdio") - async def _list_resources_operation(session: ClientSession): - return await session.list_resources() + async def _list_resources_operation(session: ClientSession) -> ListResourcesResult: + capabilities: Final = session.get_server_capabilities() + if capabilities is not None and capabilities.resources is None: + return ListResourcesResult(resources=[]) + try: + return await session.list_resources() + except McpError as error: + if error.error.code != METHOD_NOT_FOUND: + raise + verbose_logger.debug( + "MCP client list_resources is unsupported by %s: %s", self.server_url or "stdio", error + ) + return ListResourcesResult(resources=[]) try: result: Final = await self.run_with_session(_list_resources_operation) @@ -890,8 +916,19 @@ class MCPClient: """List available resource templates from the server.""" verbose_logger.debug("MCP client listing resource templates from %s", self.server_url or "stdio") - async def _list_resource_templates_operation(session: ClientSession): - return await session.list_resource_templates() + async def _list_resource_templates_operation(session: ClientSession) -> ListResourceTemplatesResult: + capabilities: Final = session.get_server_capabilities() + if capabilities is not None and capabilities.resources is None: + return ListResourceTemplatesResult(resourceTemplates=[]) + try: + return await session.list_resource_templates() + except McpError as error: + if error.error.code != METHOD_NOT_FOUND: + raise + verbose_logger.debug( + "MCP client list_resource_templates is unsupported by %s: %s", self.server_url or "stdio", error + ) + return ListResourceTemplatesResult(resourceTemplates=[]) try: result: Final = await self.run_with_session(_list_resource_templates_operation) diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index b07e5876e8b..713330a8280 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import anyio import httpx import pytest +import respx from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth from mcp import McpError from mcp.client.streamable_http import streamable_http_client @@ -1686,3 +1687,211 @@ async def test_empty_http_event_stream_uses_the_existing_request_deadline() -> N timeout=3, ) assert isinstance(as_mcp_read_timeout(caught.value), TimeoutError) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ("prompts/list", "resources/list", "resources/templates/list")) +@pytest.mark.parametrize( + "outcome", + ( + "absent", + "other_capability", + "supported", + "method_not_found", + "internal_error", + "unauthorized", + "timeout", + "initialize_not_found", + ), +) +async def test_optional_discovery_capabilities_and_errors( + method: str, outcome: str, caplog: pytest.LogCaptureFixture +) -> None: + import logging + from unittest.mock import Mock + + from mcp.types import JSONRPCRequest + + capability: Final = "prompts" if method == "prompts/list" else "resources" + field: Final = { + "prompts/list": "prompts", + "resources/list": "resources", + "resources/templates/list": "resourceTemplates", + }[method] + advertised: Final = "resources" if capability == "prompts" else "prompts" + entry: Final = { + "prompts/list": {"name": "example"}, + "resources/list": {"name": "example", "uri": "test://example"}, + "resources/templates/list": {"name": "example", "uriTemplate": "test://{name}"}, + }[method] + + def respond(request: httpx.Request) -> httpx.Response: + if request.method == "DELETE": + return httpx.Response(200) + payload: Final = JSONRPCMessage.model_validate_json(request.content).root + if not isinstance(payload, JSONRPCRequest): + return httpx.Response(202) + if outcome == "initialize_not_found": + return httpx.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "error": {"code": -32601, "message": "Initialization rejected"}, + }, + ) + if payload.method == "initialize": + return httpx.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "protocolVersion": LATEST_PROTOCOL_VERSION, + "capabilities": {} + if outcome == "absent" + else {advertised if outcome == "other_capability" else capability: {}}, + "serverInfo": {"name": "discovery", "version": "1"}, + }, + }, + ) + if outcome == "timeout": + raise httpx.ReadTimeout("Optional list timed out", request=request) + if outcome == "unauthorized": + return httpx.Response(401) + if outcome in ("method_not_found", "internal_error", "absent", "other_capability"): + return httpx.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "error": { + "code": -32603 if outcome == "internal_error" else -32601, + "message": "Optional list rejected", + }, + }, + ) + return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "result": {field: [entry]}}) + + responder: Final = Mock(side_effect=respond) + caplog.set_level(logging.DEBUG, logger="LiteLLM") + with respx.mock(base_url="https://example.com") as router: + router.route().mock(side_effect=responder) + client: Final = MCPClient(server_url="https://example.com/mcp") + operation: Final = { + "prompts/list": client.list_prompts, + "resources/list": client.list_resources, + "resources/templates/list": client.list_resource_templates, + }[method] + result: Final = await operation() + + requests: Final = tuple( + JSONRPCMessage.model_validate_json(call.args[0].content).root + for call in responder.call_args_list + if call.args[0].method == "POST" + ) + assert sum(isinstance(request, JSONRPCRequest) and request.method == method for request in requests) == ( + 0 if outcome in ("absent", "other_capability", "initialize_not_found") else 1 + ) + assert [item.name for item in result] == (["example"] if outcome == "supported" else []) + failures: Final = tuple( + record for record in caplog.records if record.name == "LiteLLM" and record.levelno >= logging.WARNING + ) + if outcome in ("internal_error", "unauthorized", "timeout", "initialize_not_found"): + assert any(record.levelno == logging.ERROR and "failed" in record.message for record in failures) + else: + assert failures == () + if outcome == "method_not_found": + assert any( + record.levelno == logging.DEBUG and "Optional list rejected" in record.message for record in caplog.records + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("supports_first", (True, False)) +async def test_optional_discovery_uses_each_sessions_capabilities(supports_first: bool) -> None: + from unittest.mock import Mock + from mcp.types import JSONRPCRequest + + capabilities: Final = iter(({"resources": {}}, {}) if supports_first else ({}, {"resources": {}})) + + def respond(request: httpx.Request) -> httpx.Response: + if request.method == "DELETE": + return httpx.Response(200) + payload: Final = JSONRPCMessage.model_validate_json(request.content).root + if not isinstance(payload, JSONRPCRequest): + return httpx.Response(202) + result: Final = ( + { + "protocolVersion": LATEST_PROTOCOL_VERSION, + "capabilities": next(capabilities), + "serverInfo": {"name": "changing", "version": "1"}, + } + if payload.method == "initialize" + else {"resources": [{"name": "example", "uri": "test://example"}]} + ) + return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "result": result}) + + responder: Final = Mock(side_effect=respond) + with respx.mock(base_url="https://example.com") as router: + router.route().mock(side_effect=responder) + client: Final = MCPClient(server_url="https://example.com/mcp") + first: Final = await client.list_resources() + second: Final = await client.list_resources() + + assert [item.name for item in first] == (["example"] if supports_first else []) + assert [item.name for item in second] == ([] if supports_first else ["example"]) + requests: Final = tuple( + JSONRPCMessage.model_validate_json(call.args[0].content).root + for call in responder.call_args_list + if call.args[0].method == "POST" + ) + assert sum(isinstance(request, JSONRPCRequest) and request.method == "resources/list" for request in requests) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ("prompts/list", "resources/list", "resources/templates/list")) +async def test_optional_discovery_preserves_cancellation(method: str) -> None: + from mcp.types import JSONRPCRequest + + ready: Final = asyncio.Event() + pending: Final = asyncio.Event() + + async def respond(request: httpx.Request) -> httpx.Response: + if request.method == "DELETE": + return httpx.Response(200) + payload: Final = JSONRPCMessage.model_validate_json(request.content).root + if not isinstance(payload, JSONRPCRequest): + return httpx.Response(202) + if payload.method == "initialize": + return httpx.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "protocolVersion": LATEST_PROTOCOL_VERSION, + "capabilities": {"resources": {}, "prompts": {}}, + "serverInfo": {"name": "pending", "version": "1"}, + }, + }, + ) + ready.set() + await pending.wait() + return httpx.Response(202) + + with respx.mock(base_url="https://example.com") as router: + router.route().mock(side_effect=respond) + client: Final = MCPClient(server_url="https://example.com/mcp") + operation: Final = { + "prompts/list": client.list_prompts, + "resources/list": client.list_resources, + "resources/templates/list": client.list_resource_templates, + }[method] + task: Final = asyncio.create_task(operation()) + try: + await asyncio.wait_for(ready.wait(), timeout=3) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=3) From a0bb021f028efa4b29ac6dca3bdbff67c6f890f0 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:41:56 -0700 Subject: [PATCH 293/411] fix(mcp): preserve gateway-owned resource scope through consent --- .../mcp_server/gateway_dcr_flow.py | 12 +++-- .../mcp_server/test_gateway_dcr_flow.py | 46 ++++++++++++++++++- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py index 3d94fa345d0..f4889008e94 100644 --- a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py +++ b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py @@ -411,7 +411,7 @@ def relative_request_url(request: Request) -> str: def resolve_scoped_resource_server(request: Request, resource: str | None) -> MCPServer | None: - """Resolve an RFC 8707 ``resource`` value to the single gateway-managed oauth2 server it + """Resolve an RFC 8707 ``resource`` value to the single gateway-owned server it names, or ``None`` for every other shape: absent, the aggregate resource, a foreign host, an unparseable value, a multi-server path, an unknown name, or any server mode the keyless gateway flow does not serve (whose protected-resource metadata never directs a @@ -443,7 +443,7 @@ def resolve_scoped_resource_server(request: Request, resource: str | None) -> MC if len(names) != 1: return None server: Final = global_mcp_server_manager.get_mcp_server_by_name(names[0]) - if server is None or not server.is_gateway_managed_oauth2: + if server is None or not (server.is_gateway_managed_oauth2 or server.advertises_gateway_authorization_server): return None return server @@ -729,11 +729,15 @@ async def _flow_target( server: Final = global_mcp_server_manager.get_mcp_server_by_id(flow.resource_server_id) if ( server is None - or not server.is_gateway_managed_oauth2 + or not (server.is_gateway_managed_oauth2 or server.advertises_gateway_authorization_server) or not await lookup_server_reachability(flow.user_id, server.server_id) ): return "stale", None - state: Final = "m2m" if MCPServerManager.effective_oauth2_flow(server) == "client_credentials" else "interactive" + state: Final = ( + "interactive" + if server.is_gateway_managed_oauth2 and MCPServerManager.effective_oauth2_flow(server) != "client_credentials" + else "m2m" + ) return state, server diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py index 73a52a8d2e8..8aa4cfc5619 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py @@ -833,7 +833,7 @@ async def test_manual_delivery_page_renders_the_url_as_data_never_as_a_shell_com assert 'value="' in body -def _scoped_mcp_server(name="github", **kw): +def _scoped_mcp_server(name="github", auth_type="oauth2", **kw): from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -844,7 +844,7 @@ def _scoped_mcp_server(name="github", **kw): alias=name, url="https://upstream.example/mcp", transport="http", - auth_type=MCPAuth.oauth2, + auth_type=MCPAuth(auth_type) if auth_type is not None else None, **kw, ) @@ -2044,3 +2044,45 @@ async def test_introspect_fails_closed_on_dead_user_and_503s_on_outage(): status, body = await _introspect(minted.token.get_secret_value(), master_key=None) assert (status, body["error"]) == (500, "server_error") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "auth_type", [None, "none", "api_key", "bearer_token", "basic", "authorization", "token", "aws_sigv4"] +) +@pytest.mark.parametrize("resource", ["https://llm.example.com/mcp/github", "https://llm.example.com/github/mcp"]) +async def test_gateway_owned_resource_stays_scoped_through_consent_and_refresh(auth_type, resource): + from unittest.mock import patch + + client_id = (await _register([REDIRECT_URI]))["client_id"] + server = _scoped_mcp_server(auth_type=auth_type) + vendor = _VendorCredential("absent") + with patch(_MANAGER_PATCH) as manager: + manager.get_mcp_server_by_name.return_value = server + response = _scoped_authorize(client_id, resource) + described = await _describe_page(response, scoped_server=server, vendor=vendor) + assert json.loads(described.body) == { + "state": "m2m", + "client_origin": "https://claude.ai", + "server_id": "github-id", + "server_name": "github", + "connected": True, + } + unreachable = await _complete_page(response, scoped_server=server, reachable=_ServerReachability(False)) + assert unreachable.status_code == 400 + cache = DualCache() + completed = await _complete_page(response, scoped_server=server, vendor=vendor, cache=cache) + assert completed.status_code == 303 + assert vendor.calls == [] + code = parse_qs(urlparse(completed.headers["location"]).query)["code"][0] + with patch(_MANAGER_PATCH) as manager: + manager.get_mcp_server_by_name.return_value = server + redeemed = await _redeem(code, client_id, cache=cache, resource=resource) + assert redeemed.status_code == 200 + payload = json.loads(redeemed.body) + assert _opened_principal(payload).resource_server_id == "github-id" + renewed = await _redeem( + None, client_id, cache=cache, grant_type="refresh_token", refresh_token=payload["refresh_token"] + ) + assert renewed.status_code == 200 + assert _opened_principal(json.loads(renewed.body)).resource_server_id == "github-id" From 95f6e96ef982d0244cfa8607d5c4fd1f2703867c Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:57:33 -0700 Subject: [PATCH 294/411] test(mcp): type auth diagnostics regression parameters --- .../outbound_credentials/test_resolver.py | 11 +++++++---- .../mcp_server/test_mcp_server_manager.py | 13 ++++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py index 2eab053ab73..5fab4ceec72 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py @@ -19,6 +19,7 @@ from pydantic import SecretStr from litellm.proxy._experimental.mcp_server.outbound_credentials import ( ApiKeyConfig, + AuthConfig, AuthorizationCodeConfig, AwsSigV4Config, Byok, @@ -1216,7 +1217,9 @@ async def test_passthrough_ignores_the_carrier_and_keeps_the_callers_slot(): (AuthorizationCodeConfig(), Subject(tenant_id="", subject_id="alice"), "stored-user-token", "Bearer stored-alice"), ], ) -async def test_resolved_source_matches_the_credential_sent_upstream(config, subject, expected_source, expected_header): +async def test_resolved_source_matches_the_credential_sent_upstream( + config: AuthConfig, subject: Subject, expected_source: str, expected_header: str | None +) -> None: from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source store = _FakeTokenStore({("alice", "s"): OAuthToken(access_token="stored-alice")}) @@ -1230,7 +1233,7 @@ async def test_resolved_source_matches_the_credential_sent_upstream(config, subj @pytest.mark.asyncio -async def test_resolved_source_preserves_missing_user_token_error(): +async def test_resolved_source_preserves_missing_user_token_error() -> None: from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source result = await resolve_credentials_with_source(UpstreamCredentialProvider(), _SUBJECT, _spec(AuthorizationCodeConfig())) @@ -1239,7 +1242,7 @@ async def test_resolved_source_preserves_missing_user_token_error(): @pytest.mark.asyncio -async def test_minted_token_sources_match_egress_and_do_not_fetch_twice(): +async def test_minted_token_sources_match_egress_and_do_not_fetch_twice() -> None: from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source source = _FakeM2MSource(Ok(OAuthToken(access_token="m2m-at"))) @@ -1259,7 +1262,7 @@ async def test_minted_token_sources_match_egress_and_do_not_fetch_twice(): @pytest.mark.asyncio -async def test_id_jag_source_describes_final_token_after_both_exchanges(): +async def test_id_jag_source_describes_final_token_after_both_exchanges() -> None: from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source endpoint = _FakeTokenEndpoint(_two_leg_ok("resource-token")) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 99447acd371..adf985e9a21 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -5,7 +5,7 @@ import logging import os import sys from datetime import datetime -from typing import Any, Dict, Final, Optional +from typing import Any, Dict, Final, Literal, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -12582,8 +12582,11 @@ async def test_pre_call_tool_check_honors_guardrail_attached_to_key(monkeypatch, ], ) async def test_debug_resolution_matches_final_header_conflict_winner( - config, extra_headers, expected_source, expected_authorization -): + config: Literal["stored", "static", "none"], + extra_headers: dict[str, str] | None, + expected_source: str, + expected_authorization: str | None, +) -> None: from mcp.server.lowlevel.server import request_ctx from mcp.shared.context import RequestContext from starlette.requests import Request @@ -12598,7 +12601,7 @@ async def test_debug_resolution_matches_final_header_conflict_winner( from litellm.types.mcp_server.mcp_server_manager import MCPServer class Store: - def __init__(self): + def __init__(self) -> None: self.calls = 0 async def fetch(self, user_id: str, server_id: str) -> OAuthToken | None: @@ -12641,7 +12644,7 @@ async def test_debug_resolution_matches_final_header_conflict_winner( @pytest.mark.asyncio @pytest.mark.parametrize("transport", ["http", "stdio"]) -async def test_debug_reports_legacy_signing_and_non_http_transport(transport): +async def test_debug_reports_legacy_signing_and_non_http_transport(transport: Literal["http", "stdio"]) -> None: from mcp.server.lowlevel.server import request_ctx from mcp.shared.context import RequestContext from starlette.requests import Request From 00b631883da02740ad285ed053d20488eba896a1 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 9 Sep 2026 22:03:16 -0700 Subject: [PATCH 295/411] fix(proxy): surface runtime-registered callbacks in UI Logging page (#38974) * fix(proxy): surface runtime-registered callbacks in /get/config/callbacks Config-file callbacks fire at runtime but never appear in the UI Logging and Alerts page because /get/config/callbacks only reads the DB-merged config. Append runtime-registered callbacks from LoggingCallbackManager as read-only rows, deduplicated against configured rows via alias normalization. UI hides edit/delete/test actions for read-only rows. * fix: filter internal proxy hooks from runtime callbacks, update test - Filter _PROXY*, ShadowEval, ServiceLogging, SkillsInjection, ResponsesID prefixes - Update test to exclude read_only rows from count assertions - Still allows deployment/guardrail callbacks to surface if configured Note: comprehensive internal-hook filtering deferred, live-pr-risk will observe real behavior on running proxy. * fix: guard non-list config callbacks in get_config, use monkeypatch in tests - Line-concat type error: normalize_callback now returns empty list for non-list types (dict/tuple/set) instead of passing through unchanged; prevents TypeError when config values are non-list - Test quality TQ005: replace manual try/finally save-restore of litellm.callbacks with monkeypatch.setattr in test_get_config_callbacks_appends_runtime_only_callbacks and test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_admin - Ruff format: wrap _internal_callback_prefixes tuple and isinstance check across multiple lines to respect 120-char limit - All three new tests pass * fix: rework runtime callback inventory filtering and dedup - Filter internal proxy hooks by name: _PROXY_ prefix plus fixed internal names (cache, _ProxyDBLogger, deployment callbacks, service hooks) - Hide guardrail instances and runtime instances of already configured callbacks via CustomLoggerRegistry class lookup - Sort runtime rows and dedup per mode for stable output - normalize_callback returns tuples for str/None/list config values and empty for any other type - Tests mock get_callbacks_by_type explicitly and pin the exact row set; UI test covers read_only action hiding * fix: list dict-shaped callback config values by their keys Dict-valued success_callback/failure_callback/callbacks settings previously listed their keys as editable rows; keep that behavior instead of dropping them to read-only runtime rows. Adds a pin test for the dict shape. * fix: mark dotted-path callbacks read-only to prevent duplicate display Configured callbacks loaded from dotted Python paths (e.g. custom_callbacks.my_logger) are never matched against runtime instances by name because the registry uses short canonical names (e.g. langsmith, arize). Mark these rows read-only to prevent the UI from attempting delete operations that would fail at the endpoint level anyway. * fix: dedupe dotted-path callbacks by instance module instead of marking them read-only A dotted-path callback loaded from config registers as an object, so it surfaces at runtime under its class name and never matched the configured string, producing a second row. Marking the config row read_only hid the duplicate but also hid delete, which does work for these rows. Match the live instance back to its configured entry by module and drop it from the runtime rows, so the callback stays a single editable row. * test: cover dotted-path dedup across success, failure, and callbacks modes * fix(proxy): filter runtime callback inventory by object identity and label read-only rows in the UI Runtime-only rows were filtered by callback name, which missed initialized CustomLogger instances, router and proxy hook methods, guardrails, and user functions. The inventory now inspects the live callback objects through a public LoggingCallbackManager.get_callback_objects accessor and hides litellm-internal hooks, guardrails, and instances of already configured callbacks. The dashboard shows a Read only label for runtime-only rows instead of an empty action cell Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): keep configured-callback assertions minimal when runtime rows are present Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): hide internal cache string callback from runtime callback inventory Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): hide auto-registered vector store hook from callback inventory Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep YAML OTel-family callbacks listed next to a configured one arize, weave_otel and langfuse_otel all initialize OpenTelemetry subclasses, so hiding runtime callbacks by configured class made one saved OTel callback swallow its YAML siblings. Match runtime instances by their own callback_name and only fall back to class identity for bare OpenTelemetry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): cover scalar and null YAML callback keys in callback inventory Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): drop docstrings that restate callback inventory helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep runtime-only s3 and sqs callbacks in UI Logging inventory _is_litellm_internal_callback checked registry membership with the display alias (s3, sqs), which is not a registry key, so runtime-only S3Logger and SQSLogger instances were classified as internal and dropped from /get/config/callbacks. Check the registered name instead and cover both loggers in the internal-exclusion regression test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../logging_callback_manager.py | 10 + litellm/proxy/proxy_server.py | 94 ++- tests/proxy_unit_tests/test_proxy_server.py | 2 +- .../proxy/proxy_server/test_routes_config.py | 564 ++++++++++++++---- .../LoggingCallbacksTable.test.tsx | 18 + .../LoggingCallbacksTableColumns.tsx | 10 + .../LoggingCallbacks/types.ts | 1 + 7 files changed, 576 insertions(+), 123 deletions(-) diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 9b612993a69..31523c9309d 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -441,7 +441,15 @@ class LoggingCallbackManager: return result + def get_callback_objects(self) -> tuple[tuple[str, CustomLogger | Callable], ...]: + return tuple( + (self._get_callback_string(callback), callback) + for callback in self._get_all_callbacks() + if not isinstance(callback, str) + ) + def _get_callback_string(self, callback: CustomLogger | Callable | str) -> str: + from litellm.integrations.opentelemetry import OpenTelemetry from litellm.litellm_core_utils.custom_logger_registry import ( CustomLoggerRegistry, ) @@ -449,6 +457,8 @@ class LoggingCallbackManager: """Convert a callback to its string representation""" if isinstance(callback, str): return callback + elif isinstance(callback, OpenTelemetry) and callback.callback_name is not None: + return callback.callback_name elif isinstance(callback, CustomLogger): # Try to get the string representation from the registry callback_str: Final = CustomLoggerRegistry.get_callback_str_from_class_type(type(callback)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 30ff47ac9f9..3bbe932f5d0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -268,7 +268,7 @@ from litellm.constants import ( WEEKLY_SPEND_REPORT_JOB_ID, ) from litellm.exceptions import RejectedRequestError -from litellm.integrations.custom_guardrail import ModifyResponseException +from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.agentic_loop_settings import ( @@ -17655,6 +17655,66 @@ async def delete_callback( ) +def _normalize_callback_alias(callback_name: str) -> str: + callback_aliases: Final = ( + ("opentelemetry", "otel"), + ("s3_v2", "s3"), + ("aws_sqs", "sqs"), + ("custom_callback_api", "generic_api"), + ) + return next( + (canonical_name for alias, canonical_name in callback_aliases if alias == callback_name), + callback_name, + ) + + +def _callback_module_name(callback: CustomLogger | Callable[..., object]) -> str: + if inspect.ismethod(callback): + return callback.__func__.__module__ + if inspect.isfunction(callback): + return callback.__module__ + return type(callback).__module__ + + +def _is_litellm_internal_callback(callback_name: str, callback: CustomLogger | Callable[..., object]) -> bool: + from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry + + module_owner: Final = _callback_module_name(callback).partition(".")[0] + is_registered_integration: Final = callback_name in CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE + return not is_registered_integration and module_owner in ("litellm", "litellm_enterprise") + + +def _is_instance_of_configured_callback( + callback_name: str, callback: CustomLogger | Callable[..., object], configured_classes: tuple[type, ...] +) -> bool: + """Self-naming OTel-family instances (`arize`, `weave_otel`) match by name, so a configured `logfire` (a bare + `OpenTelemetry`) does not hide YAML-configured siblings of the same class.""" + from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry + + class_derived_name: Final = CustomLoggerRegistry.get_callback_str_from_class_type(type(callback)) + return isinstance(callback, configured_classes) and callback_name in (class_derived_name, type(callback).__name__) + + +def _hidden_runtime_callback_names(configured_callback_names: frozenset[str]) -> frozenset[str]: + from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry + + configured_classes: Final = tuple( + CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE[name] + for name in configured_callback_names + if name in CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE + ) + configured_modules: Final = frozenset(name.rsplit(".", 1)[0] for name in configured_callback_names if "." in name) + internal_callback_names: Final = frozenset({"cache", "vector_store_pre_call_hook"}) + return internal_callback_names | frozenset( + callback_name + for callback_name, callback in litellm.logging_callback_manager.get_callback_objects() + if isinstance(callback, CustomGuardrail) + or _is_litellm_internal_callback(callback_name, callback) + or _is_instance_of_configured_callback(callback_name, callback, configured_classes) + or _callback_module_name(callback) in configured_modules + ) + + @router.get( "/get/config/callbacks", tags=["config.yaml"], @@ -17687,10 +17747,10 @@ async def get_config( # Normalize string callbacks to lists def normalize_callback(callback): if isinstance(callback, str): - return [callback] - elif callback is None: - return [] - return callback + return (callback,) + if callback is None: + return () + return tuple(callback) if isinstance(callback, (list, dict)) else () _success_callbacks = normalize_callback(_success_callbacks) _failure_callbacks = normalize_callback(_failure_callbacks) @@ -17721,6 +17781,30 @@ async def get_config( for _callback in _success_and_failure_callbacks: _data_to_return.append(process_callback(_callback, "success_and_failure", environment_variables)) + configured_callback_names: Final = frozenset( + _normalize_callback_alias(callback) + for callback in (_success_callbacks + _failure_callbacks + _success_and_failure_callbacks) + ) + runtime_callbacks_by_type: Final = litellm.logging_callback_manager.get_callbacks_by_type() + hidden_callback_names: Final = _hidden_runtime_callback_names(configured_callback_names) + runtime_callback_rows: Final = tuple( + (_normalize_callback_alias(callback_name), callback_type) + for callback_type, callback_names in ( + ("success", runtime_callbacks_by_type["success"]), + ("failure", runtime_callbacks_by_type["failure"]), + ("success_and_failure", runtime_callbacks_by_type["success_and_failure"]), + ) + for callback_name in callback_names + if callback_name not in hidden_callback_names + ) + runtime_only_rows: Final = sorted( + frozenset(row for row in runtime_callback_rows if row[0] not in configured_callback_names) + ) + _data_to_return.extend( + dict(process_callback(callback_name, callback_type, environment_variables), read_only=True) + for callback_name, callback_type in runtime_only_rows + ) + _data_to_return = _apply_callback_role_gate(_data_to_return, is_full_admin) # Check if slack alerting is on diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 54cce9cdd78..d06eb0426c9 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2920,7 +2920,7 @@ async def test_get_config_callbacks_with_all_types(client_no_auth): assert result["status"] == "success" assert "callbacks" in result - callbacks = result["callbacks"] + callbacks = [cb for cb in result["callbacks"] if not cb.get("read_only", False)] # Verify we have all 5 callbacks (2 success + 1 failure + 2 success_and_failure) assert len(callbacks) == 5 diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index dcb63b8ca82..2d9c1bd8b46 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -13,9 +13,12 @@ Routes covered: from __future__ import annotations +import asyncio import json from unittest.mock import AsyncMock, MagicMock +import pytest + from .conftest import VOLATILE_KEYS, normalize @@ -229,10 +232,7 @@ def test_config_update_no_db_error(client, auth_as, monkeypatch): json={"general_settings": {"alerting": ["slack"]}}, ) assert response.status_code != 200 - assert ( - "db" in str(response.json()).lower() - or "connect" in str(response.json()).lower() - ) + assert "db" in str(response.json()).lower() or "connect" in str(response.json()).lower() # --------------------------------------------------------------------------- @@ -273,9 +273,7 @@ def test_config_field_update_happy_admin(client, auth_as, mock_prisma, monkeypat } -def test_config_field_update_non_admin_rejected( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_update_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch): """Non-admin cannot update config fields — returns 400 with not-allowed detail (handler uses 400 for the auth gate, not 403).""" from litellm.proxy import proxy_server as ps @@ -335,9 +333,7 @@ def test_config_field_info_happy_admin(client, auth_as, mock_prisma, monkeypatch monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/field/info", params={"field_name": "max_parallel_requests"} - ) + response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"}) assert response.status_code == 200 assert normalize(response.json()) == { "field_name": "max_parallel_requests", @@ -345,9 +341,7 @@ def test_config_field_info_happy_admin(client, auth_as, mock_prisma, monkeypatch } -def test_config_field_info_non_admin_rejected( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_info_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch): """Non-admin (INTERNAL_USER) is denied — admin-view gate fires.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -356,9 +350,7 @@ def test_config_field_info_non_admin_rejected( monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.INTERNAL_USER): - response = client.get( - "/config/field/info", params={"field_name": "max_parallel_requests"} - ) + response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"}) assert response.status_code == 400 assert "error" in response.json().get("detail", {}) @@ -375,16 +367,12 @@ def test_config_field_info_field_not_in_db(client, auth_as, mock_prisma, monkeyp monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/field/info", params={"field_name": "max_parallel_requests"} - ) + response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"}) assert response.status_code == 400 assert "not in DB" in response.json().get("detail", {}).get("error", "") -def test_config_field_info_redacts_nested_secret_for_view_only_admin( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_info_redacts_nested_secret_for_view_only_admin(client, auth_as, mock_prisma, monkeypatch): """A view-only admin reading a structured field must not receive nested credentials. database_args carries aws_web_identity_token (a DynamoDB role-assumption credential); it must come back redacted while non-secret @@ -405,9 +393,7 @@ def test_config_field_info_redacts_nested_secret_for_view_only_admin( monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): - response = client.get( - "/config/field/info", params={"field_name": "database_args"} - ) + response = client.get("/config/field/info", params={"field_name": "database_args"}) assert response.status_code == 200 value = response.json()["field_value"] assert value["aws_web_identity_token"] == "REDACTED" @@ -415,9 +401,7 @@ def test_config_field_info_redacts_nested_secret_for_view_only_admin( assert value["user_table_name"] == "LiteLLM_UserTable" -def test_config_field_info_full_admin_sees_nested_secret( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_info_full_admin_sees_nested_secret(client, auth_as, mock_prisma, monkeypatch): """The redaction must not over-redact for a full PROXY_ADMIN, who needs the real nested value to populate the edit form.""" from litellm.proxy import proxy_server as ps @@ -435,18 +419,14 @@ def test_config_field_info_full_admin_sees_nested_secret( monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/field/info", params={"field_name": "database_args"} - ) + response = client.get("/config/field/info", params={"field_name": "database_args"}) assert response.status_code == 200 value = response.json()["field_value"] assert value["aws_web_identity_token"] == "sk-super-secret-token" assert value["region_name"] == "us-east-1" -def test_config_field_info_redacts_top_level_scalar_for_view_only( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_info_redacts_top_level_scalar_for_view_only(client, auth_as, mock_prisma, monkeypatch): """The top-level scalar branch must also redact for a view-only admin. database_url carries DB credentials and is not caught by the name masker, so it is in the explicit secret set.""" @@ -460,9 +440,7 @@ def test_config_field_info_redacts_top_level_scalar_for_view_only( monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): - response = client.get( - "/config/field/info", params={"field_name": "database_url"} - ) + response = client.get("/config/field/info", params={"field_name": "database_url"}) assert response.status_code == 200 assert response.json()["field_value"] == "REDACTED" @@ -476,17 +454,12 @@ def test_redact_general_setting_value_recurses_list_of_dicts(): {"path": "/foo", "headers": {"Authorization": "Bearer sk-x"}}, {"path": "/bar", "client_secret": "sk-y"}, ] - redacted = ps._redact_general_setting_value( - "some_list_field", value, is_full_admin=False - ) + redacted = ps._redact_general_setting_value("some_list_field", value, is_full_admin=False) assert redacted[0]["headers"]["Authorization"] == "REDACTED" assert redacted[0]["path"] == "/foo" assert redacted[1]["client_secret"] == "REDACTED" assert redacted[1]["path"] == "/bar" - assert ( - ps._redact_general_setting_value("some_list_field", value, is_full_admin=True) - == value - ) + assert ps._redact_general_setting_value("some_list_field", value, is_full_admin=True) == value def test_redact_secret_values_in_obj_fails_closed_at_max_depth(): @@ -504,22 +477,16 @@ def test_redact_secret_values_in_obj_fails_closed_at_max_depth(): for _ in range(ps._REDACT_SECRET_MAX_DEPTH + 2): nested = {"wrap": nested} - out = ps._redact_general_setting_value( - "some_struct_field", nested, is_full_admin=False - ) + out = ps._redact_general_setting_value("some_struct_field", nested, is_full_admin=False) # the secret must not survive anywhere in the returned tree assert "sk-leak-bottom" not in repr(out) # full admin is unaffected by the cap — the value comes back untouched - admin_out = ps._redact_general_setting_value( - "some_struct_field", nested, is_full_admin=True - ) + admin_out = ps._redact_general_setting_value("some_struct_field", nested, is_full_admin=True) assert admin_out is nested -def test_config_list_redacts_pass_through_secret_for_view_only( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_list_redacts_pass_through_secret_for_view_only(client, auth_as, mock_prisma, monkeypatch): """/config/list must not leak pass_through_endpoints upstream credentials to a view-only admin. pass_through_endpoints is a known secret-bearing field, so a non-admin gets it redacted; a full admin still sees it.""" @@ -546,24 +513,16 @@ def test_config_list_redacts_pass_through_secret_for_view_only( ) def _pass_through_value(body): - return next( - entry["field_value"] - for entry in body - if entry["field_name"] == "pass_through_endpoints" - ) + return next(entry["field_value"] for entry in body if entry["field_name"] == "pass_through_endpoints") with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): - view_resp = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + view_resp = client.get("/config/list", params={"config_type": "general_settings"}) assert view_resp.status_code == 200 assert "sk-UPSTREAM-SECRET" not in view_resp.text assert _pass_through_value(view_resp.json()) == "REDACTED" with auth_as(LitellmUserRoles.PROXY_ADMIN): - admin_resp = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + admin_resp = client.get("/config/list", params={"config_type": "general_settings"}) assert admin_resp.status_code == 200 admin_value = _pass_through_value(admin_resp.json()) assert admin_value[0]["headers"]["Authorization"] == "Bearer sk-UPSTREAM-SECRET" @@ -587,9 +546,7 @@ def test_config_list_happy_admin(client, auth_as, mock_prisma, monkeypatch): monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + response = client.get("/config/list", params={"config_type": "general_settings"}) assert response.status_code == 200 body = response.json() assert isinstance(body, list) @@ -695,9 +652,7 @@ def test_config_list_non_admin_rejected(client, auth_as, mock_prisma, monkeypatc monkeypatch.setattr(ps, "prisma_client", mock_prisma) with auth_as(LitellmUserRoles.INTERNAL_USER): - response = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + response = client.get("/config/list", params={"config_type": "general_settings"}) assert response.status_code == 400 assert "role" in response.json().get("detail", {}).get("error", "").lower() @@ -710,9 +665,7 @@ def test_config_list_no_db_error(client, auth_as, monkeypatch): monkeypatch.setattr(ps, "prisma_client", None) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.get( - "/config/list", params={"config_type": "general_settings"} - ) + response = client.get("/config/list", params={"config_type": "general_settings"}) assert response.status_code == 400 assert "error" in response.json().get("detail", {}) @@ -756,9 +709,7 @@ def test_config_field_delete_happy_admin(client, auth_as, mock_prisma, monkeypat } -def test_config_field_delete_non_admin_rejected( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_delete_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch): """Non-admin caller hits the 400 not-allowed branch with role in detail.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -778,9 +729,7 @@ def test_config_field_delete_non_admin_rejected( assert "role" in response.json().get("detail", {}).get("error", "").lower() -def test_config_field_delete_field_not_in_config( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_field_delete_field_not_in_config(client, auth_as, mock_prisma, monkeypatch): """If there is no general_settings row at all, returns 400 'not in config'.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -825,9 +774,7 @@ def test_config_callback_delete_happy_admin(client, auth_as, mock_prisma, monkey monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.post( - "/config/callback/delete", json={"callback_name": "langfuse"} - ) + response = client.post("/config/callback/delete", json={"callback_name": "langfuse"}) assert response.status_code == 200 # `deleted_at` is an ISO timestamp generated at request time — extend # the volatile set just for this assertion so dict-equality still works. @@ -840,9 +787,7 @@ def test_config_callback_delete_happy_admin(client, auth_as, mock_prisma, monkey } -def test_config_callback_delete_non_admin_rejected( - client, auth_as, mock_prisma, monkeypatch -): +def test_config_callback_delete_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch): """Non-admin caller is rejected with 400 not-allowed.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -852,9 +797,7 @@ def test_config_callback_delete_non_admin_rejected( monkeypatch.setattr(ps, "store_model_in_db", True) with auth_as(LitellmUserRoles.INTERNAL_USER): - response = client.post( - "/config/callback/delete", json={"callback_name": "langfuse"} - ) + response = client.post("/config/callback/delete", json={"callback_name": "langfuse"}) assert response.status_code == 400 assert "role" in response.json().get("detail", {}).get("error", "").lower() @@ -869,22 +812,15 @@ def test_config_callback_delete_not_found(client, auth_as, mock_prisma, monkeypa monkeypatch.setattr(ps, "store_model_in_db", True) fake_proxy_config = MagicMock() - fake_proxy_config.get_config = AsyncMock( - return_value={"litellm_settings": {"success_callback": ["slack"]}} - ) + fake_proxy_config.get_config = AsyncMock(return_value={"litellm_settings": {"success_callback": ["slack"]}}) monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) with auth_as(LitellmUserRoles.PROXY_ADMIN): - response = client.post( - "/config/callback/delete", json={"callback_name": "langfuse"} - ) + response = client.post("/config/callback/delete", json={"callback_name": "langfuse"}) # The handler re-raises HTTPException(404) verbatim (only generic # `Exception` becomes a 500 ProxyException), so pin 404 strictly. assert response.status_code == 404 - assert ( - "langfuse" in str(response.json()).lower() - or "not found" in str(response.json()).lower() - ) + assert "langfuse" in str(response.json()).lower() or "not found" in str(response.json()).lower() # --------------------------------------------------------------------------- @@ -948,10 +884,7 @@ def test_get_config_callbacks_internal_error(client, auth_as, mock_prisma, monke with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/get/config/callbacks") assert response.status_code >= 400 - assert ( - "boom" in str(response.json()).lower() - or "error" in str(response.json()).lower() - ) + assert "boom" in str(response.json()).lower() or "error" in str(response.json()).lower() _CALLBACK_ENV_FIXTURE = { @@ -985,14 +918,10 @@ def _install_callbacks_config(monkeypatch, mock_prisma): def _callback_variables(body: dict, name: str) -> dict: - return next( - cb["variables"] for cb in body["callbacks"] if cb["name"] == name - ) + return next(cb["variables"] for cb in body["callbacks"] if cb["name"] == name) -def test_get_config_callbacks_redacts_secret_env_vars_for_view_only_admin( - client, auth_as, mock_prisma, monkeypatch -): +def test_get_config_callbacks_redacts_secret_env_vars_for_view_only_admin(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy._types import LitellmUserRoles _install_callbacks_config(monkeypatch, mock_prisma) @@ -1024,9 +953,7 @@ def test_get_config_callbacks_redacts_secret_env_vars_for_view_only_admin( assert otel_vars["OTEL_ENDPOINT"] == _CALLBACK_ENV_FIXTURE["OTEL_ENDPOINT"] -def test_get_config_callbacks_full_admin_still_sees_secret_env_vars( - client, auth_as, mock_prisma, monkeypatch -): +def test_get_config_callbacks_full_admin_still_sees_secret_env_vars(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy._types import LitellmUserRoles _install_callbacks_config(monkeypatch, mock_prisma) @@ -1047,9 +974,7 @@ def test_get_config_callbacks_full_admin_still_sees_secret_env_vars( assert otel_vars["OTEL_HEADERS"] == _CALLBACK_ENV_FIXTURE["OTEL_HEADERS"] -def test_get_config_callbacks_redacts_slack_webhook_urls_for_view_only_admin( - client, auth_as, mock_prisma, monkeypatch -): +def test_get_config_callbacks_redacts_slack_webhook_urls_for_view_only_admin(client, auth_as, mock_prisma, monkeypatch): from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -1170,6 +1095,413 @@ def test_get_config_callbacks_redacts_email_alerting_vars_for_view_only_admin( assert admin_email["SMTP_HOST"] == "smtp.resend.com" +def test_get_config_callbacks_appends_runtime_only_callbacks(client, auth_as, mock_prisma, monkeypatch): + """LIT-5281: a YAML callback that the DB callback list replaced in the merged config still runs, so it must + show up as a read_only row next to the editable DB-configured one.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": ["langfuse"]}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + import litellm + from litellm.integrations.langsmith import LangsmithLogger + from litellm.integrations.opentelemetry import OpenTelemetry + + monkeypatch.setattr(litellm, "success_callback", ["langfuse", LangsmithLogger()]) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "callbacks", [OpenTelemetry()]) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + + assert [(cb["name"], cb["type"], cb.get("read_only", False)) for cb in response.json()["callbacks"]] == [ + ("langfuse", "success", False), + ("langsmith", "success", True), + ("otel", "success_and_failure", True), + ] + + +def test_get_config_callbacks_accepts_scalar_and_null_yaml_callbacks(client, auth_as, mock_prisma, monkeypatch): + """`success_callback: langfuse` (a YAML scalar) is one configured callback, not eight single-letter ones, and a + `callbacks: null` key contributes nothing.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": "langfuse", "failure_callback": None, "callbacks": None}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + import litellm + from litellm.integrations.langsmith import LangsmithLogger + + monkeypatch.setattr(litellm, "success_callback", ["langfuse", LangsmithLogger()]) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "callbacks", []) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + + assert [(cb["name"], cb["type"], cb.get("read_only", False)) for cb in response.json()["callbacks"]] == [ + ("langfuse", "success", False), + ("langsmith", "success", True), + ] + + +def test_get_config_callbacks_deduplicates_configured_and_runtime(client, auth_as, mock_prisma, monkeypatch): + """A configured callback shows once as editable, whether the runtime holds its string or an initialized instance + (arize initializes an ArizeLogger, logfire a bare OpenTelemetry that only its class identifies).""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": ["langfuse", "arize", "logfire"]}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + import litellm + from litellm.integrations.arize.arize import ArizeLogger + from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig + + arize_logger = ArizeLogger(config=OpenTelemetryConfig(exporter="console"), callback_name="arize") + monkeypatch.setattr(litellm, "success_callback", ["langfuse", arize_logger, OpenTelemetry()]) + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + + assert [(cb["name"], cb["type"], cb.get("read_only", False)) for cb in response.json()["callbacks"]] == [ + ("langfuse", "success", False), + ("arize", "success", False), + ("logfire", "success", False), + ] + + +def test_get_config_callbacks_keeps_yaml_otel_family_callbacks_next_to_configured_one( + client, auth_as, mock_prisma, monkeypatch +): + """LIT-5281: arize, weave_otel and langfuse_otel all initialize OpenTelemetry subclasses. Saving one of them + from the dashboard replaces the YAML `callbacks` list, so the YAML siblings keep running and must stay listed + under their own names instead of being hidden as duplicates of the configured OTel callback.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"callbacks": ["langfuse_otel"]}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + import litellm + from litellm.integrations.arize.arize import ArizeLogger + from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger + from litellm.integrations.langsmith import LangsmithLogger + from litellm.integrations.opentelemetry import OpenTelemetryConfig + from litellm.integrations.weave.weave_otel import WeaveOtelLogger + + console_config = OpenTelemetryConfig(exporter="console") + monkeypatch.setattr(litellm, "success_callback", [LangsmithLogger()]) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr( + litellm, + "callbacks", + [ + ArizeLogger(config=console_config, callback_name="arize"), + WeaveOtelLogger(config=console_config), + LangfuseOtelLogger(config=console_config, callback_name="langfuse_otel"), + ], + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + + assert [(cb["name"], cb["type"], cb.get("read_only", False)) for cb in response.json()["callbacks"]] == [ + ("langfuse_otel", "success_and_failure", False), + ("arize", "success_and_failure", True), + ("langsmith", "success", True), + ("weave_otel", "success_and_failure", True), + ] + + +def _dotted_path_test_function(*args, **kwargs): + pass + + +@pytest.mark.parametrize("handler_kind", ["instance", "function"]) +@pytest.mark.parametrize( + "config_key,expected_type", + [ + ("success_callback", "success"), + ("failure_callback", "failure"), + ("callbacks", "success_and_failure"), + ], +) +def test_get_config_callbacks_deduplicates_dotted_path_callback( + client, auth_as, mock_prisma, monkeypatch, config_key, expected_type, handler_kind +): + """A dotted-path callback stays a single editable row instead of duplicating under its class or function name.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + import litellm + from litellm.integrations.custom_logger import CustomLogger + + class _DottedPathTestHandler(CustomLogger): + pass + + dotted_handler = _DottedPathTestHandler() if handler_kind == "instance" else _dotted_path_test_function + dotted_path = f"{__name__}.dotted_handler" + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {config_key: [dotted_path]}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + monkeypatch.setattr(litellm, "callbacks", [dotted_handler]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + + assert response.status_code == 200 + callbacks = response.json()["callbacks"] + assert [(callback["name"], callback["type"], callback.get("read_only", False)) for callback in callbacks] == [ + (dotted_path, expected_type, False) + ] + + +def test_get_config_callbacks_lists_dict_shaped_config_callbacks(client, auth_as, mock_prisma, monkeypatch): + """Dict-shaped success_callback config values list their keys as editable rows.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": {"langsmith": {"batch_size": 1}}}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + import litellm + + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_callbacks_by_type", + MagicMock(return_value={"success": ["langsmith"], "failure": [], "success_and_failure": []}), + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + + assert response.status_code == 200 + callbacks = response.json()["callbacks"] + assert [(callback["name"], callback.get("read_only", False)) for callback in callbacks] == [("langsmith", False)] + + +def test_get_config_callbacks_excludes_internal_runtime_callbacks(client, auth_as, mock_prisma, monkeypatch): + """Proxy infrastructure callbacks are excluded from callback inventory.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": []}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles + + import litellm + from litellm._service_logger import ServiceLogging + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.integrations.custom_logger import CustomLogger + from litellm.integrations.langsmith import LangsmithLogger + from litellm.integrations.s3_v2 import S3Logger + from litellm.integrations.sqs import SQSLogger + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import VectorStorePreCallHook + from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter + from litellm.router import Router + + class _InventoryTestGuardrail(CustomGuardrail): + pass + + class _UserCodeLogger(CustomLogger): + pass + + def user_code_function(*args, **kwargs): + pass + + async def build_aws_loggers() -> tuple[S3Logger, SQSLogger]: + return S3Logger(s3_bucket_name="inventory-bucket"), SQSLogger(sqs_queue_url="https://sqs.example/inventory") + + s3_logger, sqs_logger = asyncio.run(build_aws_loggers()) + router = Router(model_list=[]) + monkeypatch.setattr(litellm, "input_callback", []) + monkeypatch.setattr( + litellm, "success_callback", [LangsmithLogger(), s3_logger, router.sync_deployment_callback_on_success] + ) + monkeypatch.setattr(litellm, "_async_success_callback", [sqs_logger, router.deployment_callback_on_success]) + monkeypatch.setattr(litellm, "failure_callback", [user_code_function]) + monkeypatch.setattr(litellm, "_async_failure_callback", [router.async_deployment_callback_on_failure]) + monkeypatch.setattr( + litellm, + "callbacks", + [ + _PROXY_MaxBudgetLimiter(), + _PROXY_LiteLLMManagedFiles(internal_usage_cache=MagicMock(), prisma_client=MagicMock()), + ServiceLogging(), + VectorStorePreCallHook(), + _InventoryTestGuardrail(guardrail_name="inventory-test-guardrail"), + _UserCodeLogger(), + ], + ) + monkeypatch.setattr(litellm, "cache", litellm.Cache(type="local")) + assert "cache" in litellm.success_callback and "cache" in litellm._async_success_callback + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/get/config/callbacks") + + assert response.status_code == 200 + assert [ + (callback["name"], callback["type"], callback["read_only"]) for callback in response.json()["callbacks"] + ] == [ + ("_UserCodeLogger", "success_and_failure", True), + ("langsmith", "success", True), + ("s3", "success", True), + ("sqs", "success", True), + ("user_code_function", "failure", True), + ] + + +def test_get_config_callbacks_redacts_runtime_only_row_secrets_for_view_only_admin( + client, auth_as, mock_prisma, monkeypatch +): + """Runtime-only callback rows are subject to the same redaction gate as configured.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "llm_router", None) + + fake_proxy_config = MagicMock() + fake_proxy_config.get_config = AsyncMock( + return_value={ + "litellm_settings": {"success_callback": []}, + "general_settings": {}, + "environment_variables": dict(_CALLBACK_ENV_FIXTURE), + } + ) + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + import litellm + + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "callbacks", ["otel"]) + + with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + response = client.get("/get/config/callbacks") + assert response.status_code == 200 + body = response.json() + + callbacks = body["callbacks"] + otel_cb = next((cb for cb in callbacks if cb["name"] == "otel"), None) + assert otel_cb is not None + assert otel_cb["type"] == "success_and_failure" + assert otel_cb["read_only"] is True + assert otel_cb["variables"]["OTEL_HEADERS"] == "REDACTED" + assert otel_cb["variables"]["OTEL_ENDPOINT"] == _CALLBACK_ENV_FIXTURE["OTEL_ENDPOINT"] + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + admin_response = client.get("/get/config/callbacks") + assert admin_response.status_code == 200 + admin_body = admin_response.json() + admin_otel = next((cb for cb in admin_body["callbacks"] if cb["name"] == "otel"), None) + assert admin_otel is not None + assert admin_otel["variables"]["OTEL_HEADERS"] == _CALLBACK_ENV_FIXTURE["OTEL_HEADERS"] + + # --------------------------------------------------------------------------- # GET /config/yaml # --------------------------------------------------------------------------- @@ -1183,9 +1515,7 @@ def test_config_yaml_returns_demo_payload(client, auth_as): response = client.request("GET", "/config/yaml", json={}) shape = { "status": response.status_code, - "media_type_yaml": response.headers.get("content-type", "").startswith( - "application/json" - ), + "media_type_yaml": response.headers.get("content-type", "").startswith("application/json"), "has_body": len(response.content) > 0, } assert shape == { diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx index 3ef7dee01b0..a4d76e2cc06 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.test.tsx @@ -97,6 +97,24 @@ describe("LoggingCallbacksTable", () => { expect(onDelete).toHaveBeenCalledWith(callback); }); + it("shows a read-only label instead of the actions menu for runtime-only callback rows", () => { + render( + , + ); + expect(screen.getByTestId("callback-actions-langfuse-success")).toBeInTheDocument(); + expect(screen.queryByTestId("callback-actions-datadog-success")).not.toBeInTheDocument(); + expect(screen.getAllByText("Read only")).toHaveLength(1); + }); + // Regression: `/get_callbacks` returns the same `name` twice when a // callback is registered for both success and failure (e.g. `generic_api` // → POST to spend-log on both 200 and 4xx/5xx). The UI used to ignore diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx index 2263fe03b3d..950ef5de6e4 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTableColumns.tsx @@ -50,6 +50,16 @@ interface CallbackRowActionsProps { } function CallbackRowActions({ callback, onTest, onEdit, onDelete }: CallbackRowActionsProps) { + if (callback.read_only) { + return ( + + Read only + + ); + } return ( Date: Wed, 9 Sep 2026 22:04:37 -0700 Subject: [PATCH 296/411] feat(proxy): expose reversible Claude Code model listing aliases (#40515) Encode complete non-Claude source names and include source_model in the Claude Code listing. Preserve configured route and alias precedence, normalize once before model policy checks, and select CLI models using explicit source identity instead of name stripping or positional joins. Resolves LIT-7360 Claude-Session: https://claude.ai/code/session_01WyqeRhfZGm26zAnHx9P3kq Co-authored-by: Claude Fable 5.1 --- litellm/llms/anthropic/common_utils.py | 27 +++- litellm/proxy/auth/user_api_key_auth.py | 41 ++++++ litellm/proxy/client/cli/README.md | 4 +- .../proxy/client/cli/commands/configure.py | 68 +++++++--- litellm/proxy/client/cli/commands/pi.py | 50 +++++-- .../proxy/common_utils/model_listing_utils.py | 123 +++++++++++++++++- litellm/proxy/proxy_server.py | 40 ++++++ .../anthropic/test_anthropic_common_utils.py | 22 ++++ .../proxy/auth/test_user_api_key_auth.py | 43 ++++++ .../client/cli/test_configure_commands.py | 73 ++++++++++- .../test_litellm/proxy/client/cli/test_pi.py | 38 ++++++ .../common_utils/test_model_listing_utils.py | 109 ++++++++++++++++ .../proxy/proxy_server/test_routes_models.py | 50 +++++++ 13 files changed, 647 insertions(+), 41 deletions(-) create mode 100644 tests/test_litellm/proxy/common_utils/test_model_listing_utils.py diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index ca520116606..87c4ec8938e 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -73,8 +73,13 @@ _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") + + def is_claude_code_user_agent(user_agent: str) -> bool: - return user_agent.startswith("claude-cli/") + """Claude Code sends its API calls through the Anthropic SDK as `claude-cli/` and its own + fetches, such as gateway model discovery, as `claude-code/`""" + return user_agent.startswith(_CLAUDE_CODE_USER_AGENT_PREFIXES) def _validated_claude_code_mapping(value: object) -> dict[object, object] | None: @@ -1656,11 +1661,16 @@ def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: def _anthropic_model_entry( - model: ModelInfoResponse, created_at: str, display_names: Mapping[str, str] + model: ModelInfoResponse, created_at: str, display_names: Mapping[str, str], listed_ids: Mapping[str, str] ) -> Mapping[str, object]: + listed_id: Final = listed_ids.get(model["id"]) + source: Final[Mapping[str, object]] = ( + MappingProxyType({"source_model": model["id"]}) if listed_id is not None else MappingProxyType({}) + ) return { # mutable-ok: JSON response body, serialized by the route and never mutated "type": "model", - "id": model["id"], + "id": listed_id or model["id"], + **source, "display_name": display_names.get(model["id"], model["id"]), "created_at": created_at, "max_input_tokens": model.get("max_input_tokens"), @@ -1671,6 +1681,7 @@ def _anthropic_model_entry( def create_anthropic_model_list_response( models: Sequence[ModelInfoResponse], display_names: Mapping[str, str] = MappingProxyType({}), + listed_ids: Mapping[str, str] = MappingProxyType({}), ) -> Mapping[str, object]: """Build the Anthropic-native /v1/models envelope. @@ -1680,17 +1691,19 @@ def create_anthropic_model_list_response( over from the OpenAI-shaped listing, named as the Messages API names them, and are always present because the vendor shape declares them nullable, not optional. display_names maps a listed model id to a configured human-readable name; ids - without an entry fall back to the id itself, matching the vendor behavior + without an entry fall back to the id itself, matching the vendor behavior. + listed_ids maps a model id to the id the caller should see it under (the Claude + Code view); ids without an entry are listed as they are """ created_at: Final = ( datetime.fromtimestamp(DEFAULT_MODEL_CREATED_AT_TIME, tz=timezone.utc).isoformat().replace("+00:00", "Z") ) data: Final = [ # mutable-ok: JSON response body, serialized by the route and never mutated - _anthropic_model_entry(model, created_at, display_names) for model in models + _anthropic_model_entry(model, created_at, display_names, listed_ids) for model in models ] return { # mutable-ok: JSON response body, serialized by the route and never mutated "data": data, "has_more": False, - "first_id": models[0]["id"] if models else None, - "last_id": models[-1]["id"] if models else None, + "first_id": data[0]["id"] if data else None, + "last_id": data[-1]["id"] if data else None, } diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index bfccb703e76..64c0da1c28f 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -92,6 +92,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_set_request_parsed_body, populate_request_with_path_params, ) +from litellm.proxy.common_utils.model_listing_utils import claude_code_requested_group from litellm.proxy.common_utils.realtime_utils import _realtime_request_body from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, @@ -183,6 +184,44 @@ def _get_model_from_request_context( ) +_CLAUDE_MODEL_ROUTES: Final = frozenset( + f"/{prefix}{endpoint}" for prefix in ("", "v1/") for endpoint in ("messages", "chat/completions", "responses") +) +_CLAUDE_MODEL_NORMALIZED: Final = "litellm.claude_model_normalized" + + +async def _normalize_claude_model( + request_data: dict, valid_token: UserAPIKeyAuth, request: Request | None, route: str +) -> None: + from litellm.proxy.proxy_server import llm_router, prisma_client, proxy_config, proxy_logging_obj + + if route not in _CLAUDE_MODEL_ROUTES or llm_router is None: + return + if request is not None and request.scope.get(_CLAUDE_MODEL_NORMALIZED) is True: + return + requested: Final = _get_model_from_request_context(request_data, route, request, llm_router) + if not isinstance(requested, str) or requested != request_data.get("model"): + return + if not requested.startswith("claude-router-") and not requested.lower().endswith("[1m]"): + return + settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict=valid_token, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) + aliases: Final = settings.get("model_group_alias") if isinstance(settings, Mapping) else None + source: Final = claude_code_requested_group( + requested, llm_router, valid_token.team_id, (valid_token.aliases, valid_token.team_model_aliases, aliases) + ) + if request is not None: + request.scope[_CLAUDE_MODEL_NORMALIZED] = True + if source is None: + return + request_data["model"] = source + _safe_set_request_parsed_body(request=request, parsed_body=request_data) + if request is not None: + request._json = request_data + request._body = orjson.dumps(request_data) + + def _get_model_names_for_budget_checks( model: str | list[str] | None, ) -> list[str]: @@ -2768,6 +2807,7 @@ async def _authorize_authenticated_request( """ ## ENSURE DISABLE ROUTE WORKS ACROSS ALL USER AUTH FLOWS ## RouteChecks.should_call_route(route=route, valid_token=user_api_key_auth_obj, request=request) + await _normalize_claude_model(request_data, user_api_key_auth_obj, request, route) # Single authorization point. Builder paths MUST NOT call common_checks. # Route through the same exception handler the builder uses so @@ -3131,6 +3171,7 @@ async def _enforce_key_and_fallback_model_access( Key-level model allowlist and client fallbacks (same as standard auth). Not included in common_checks — common_checks enforces team/user/project model access only. """ + await _normalize_claude_model(request_data, valid_token, request, route) config: Final = valid_token.config if config != {}: diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 03d01ff7f66..4045857a237 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -490,7 +490,7 @@ lite codex exec "summarize the repo" Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process. -The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, and older versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. +The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. Your own flags come after the injected pin, so `lite pi --model litellm/` wins, and inside the TUI the `/model` picker lists every synced litellm model. @@ -548,7 +548,7 @@ lite --base-url https://your-proxy.example.com configure claude --api-key sk-... claude ``` -With `--api-key` (or `lite --api-key` / `LITELLM_PROXY_API_KEY`) the key is written into `env.ANTHROPIC_AUTH_TOKEN`. Without one, your `lite login` credential is used the way `--config-claude` uses it, through `apiKeyHelper`, so a later `lite login` (or a `--pkce` renewal) picks up on its own and nothing secret lands in the file; a missing or stale login is refreshed first. Either way the command checks the key against `GET /v1/models`, then patches `~/.claude/settings.json`: `env.ANTHROPIC_BASE_URL`, the credential, and `env.ENABLE_TOOL_SEARCH` and `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` when those are missing, so Claude Code's `/model` picker lists the proxy's models (the ones whose id contains `claude` or `anthropic`) and you pick between them as usual. Claude Code keeps its own default model until you switch, so that id has to exist on the proxy for the first message to go through; `--model` (or the interactive prompt below) sets the model Claude Code starts on instead, as the top-level `model` key, which has to be on `/v1/models` for the key. Nothing forces Claude Code's sub-agent or background tiers onto a proxy model, so those built-in ids need to exist on the proxy too; `lite autoroute up` is the mode that pins every tier to one group. Claude Code treats a name it does not know as an unknown model: it prints a one-line `unrecognized_model` note, assumes a 200k context window and sends no thinking parameters for it, so either name the group like a Claude model id or append `[1m]` to opt into the 1M window. The other credential slots (`env.ANTHROPIC_API_KEY`, a stale `env.ANTHROPIC_AUTH_TOKEN` or `apiKeyHelper`) are removed so they cannot fight the one written. Every other setting is preserved and the file is written atomically with owner-only permissions; if `settings.json` is a symlink into a dotfiles repository, the key is written through to that target and the command says so, so keep it out of version control +With `--api-key` (or `lite --api-key` / `LITELLM_PROXY_API_KEY`) the key is written into `env.ANTHROPIC_AUTH_TOKEN`. Without one, your `lite login` credential is used the way `--config-claude` uses it, through `apiKeyHelper`, so a later `lite login` (or a `--pkce` renewal) picks up on its own and nothing secret lands in the file; a missing or stale login is refreshed first. Either way the command checks the key against `GET /v1/models`, then patches `~/.claude/settings.json`: `env.ANTHROPIC_BASE_URL`, the credential, and `env.ENABLE_TOOL_SEARCH` and `env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY` when those are missing, so Claude Code's `/model` picker lists the proxy's models (under `claude-router-` for a group whose id contains neither `claude` nor `anthropic`, since Claude Code lists only those) and you pick between them as usual. Claude Code keeps its own default model until you switch, so that id has to exist on the proxy for the first message to go through; `--model` (or the interactive prompt below) sets the model Claude Code starts on instead, as the top-level `model` key, which has to be on `/v1/models` for the key. Nothing forces Claude Code's sub-agent or background tiers onto a proxy model, so those built-in ids need to exist on the proxy too; `lite autoroute up` is the mode that pins every tier to one group. Claude Code treats a name it does not know as an unknown model: it prints a one-line `unrecognized_model` note, assumes a 200k context window (the proxy appends `[1m]` for a group whose configured or known input window reaches 1M) and sends no thinking parameters for it, so name the group like a Claude model id to change that. The other credential slots (`env.ANTHROPIC_API_KEY`, a stale `env.ANTHROPIC_AUTH_TOKEN` or `apiKeyHelper`) are removed so they cannot fight the one written. Every other setting is preserved and the file is written atomically with owner-only permissions; if `settings.json` is a symlink into a dotfiles repository, the key is written through to that target and the command says so, so keep it out of version control Plain `lite configure`, with no agent named, asks the same things interactively: which agents to wire (Claude Code today) and which of the proxy's models to start on, picked from `/v1/models` with a type-to-filter prompt diff --git a/litellm/proxy/client/cli/commands/configure.py b/litellm/proxy/client/cli/commands/configure.py index 83fbd323924..9d329f8d8f2 100644 --- a/litellm/proxy/client/cli/commands/configure.py +++ b/litellm/proxy/client/cli/commands/configure.py @@ -1,16 +1,23 @@ """`lite configure claude` and `lite unconfigure claude`: persistent Claude Code wiring, undoable.""" import os -import re import sys from collections.abc import Callable, Sequence +from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import Final import click from InquirerPy import inquirer from InquirerPy.base.control import Choice +from litellm.proxy.common_utils.model_listing_utils import ( + CLAUDE_CODE_CLIENT, + CLAUDE_CODE_PICKER_PATTERN, + GATEWAY_CLIENT_HEADER, +) + from .auth import CliContextObj, context_secret_vault, get_stored_api_key from .claude_settings import ( STARTING_MODEL_ROLE, @@ -30,14 +37,16 @@ from .claude_settings import ( settings_file_owners, unconfigure_claude_settings, ) -from .pi import ListingFailure, PiSyncError, fetch_model_ids +from .pi import ListedModel, ListingFailure, PiSyncError, fetch_model_listing from .up import ensure_fresh_login _LISTED_MODELS_SHOWN: Final = 20 _CLAUDE_TARGET: Final = "claude" _TARGETS: Final = ((_CLAUDE_TARGET, "Claude Code (CLI)"),) _KEEP_DEFAULT_MODEL: Final = "Keep Claude Code's own default" -_CLAUDE_CODE_PICKER_FILTER: Final = re.compile(r"claude|anthropic", re.IGNORECASE) +_CLAUDE_CODE_VIEW: Final = MappingProxyType( + {"anthropic-version": "2023-06-01", GATEWAY_CLIENT_HEADER: CLAUDE_CODE_CLIENT} +) _MODEL_OPTION_HELP: Final = ( f"Proxy model to set as {STARTING_MODEL_ROLE}. Must be listed on /v1/models for the key; without it, " "Claude Code keeps its own default and a pin an earlier configure made is let go of. Nothing pins Claude " @@ -65,7 +74,16 @@ def resolve_credential(ctx: click.Context, api_key: str | None) -> tuple[ClaudeC return ApiKeyHelper(resolve_api_key_helper(base_url)), stored -def _start(ctx: click.Context, api_key: str | None) -> tuple[ClaudeCredential, tuple[str, ...]]: +@dataclass(frozen=True, slots=True) +class _Listing: + models: tuple[ListedModel, ...] + + @property + def ids(self) -> tuple[str, ...]: + return tuple(model.id for model in self.models) + + +def _start(ctx: click.Context, api_key: str | None) -> tuple[ClaudeCredential, _Listing]: """Every configure path begins the same way: the local ownership check first, so a `lite up` session is refused before any login prompt or request, then the credential, then the listing.""" settings_path: Final = claude_settings_path(os.environ) @@ -88,21 +106,28 @@ def _listing_error(base_url: str, error: PiSyncError) -> str: return f"{error.message} The proxy at {base_url} answered, so check that it is a LiteLLM proxy and is healthy." -def _listed_models(base_url: str, key: str) -> tuple[str, ...]: - listed: Final = fetch_model_ids(base_url, key) +def _listed_models(base_url: str, key: str) -> _Listing: + listed: Final = fetch_model_listing(base_url, key, headers=_CLAUDE_CODE_VIEW) if isinstance(listed, PiSyncError): raise click.ClickException(_listing_error(base_url, listed)) - return listed + return _Listing(listed) + + +def _starting_model(model: str, listing: _Listing) -> str | None: + source: Final = next((listed.id for listed in listing.models if listed.source_model == model), None) + return source or next((listed.id for listed in listing.models if listed.id == model), None) def _model_choice(model: str | None) -> ModelChoice: return StartOn(model) if model is not None else UnpinModel() -def _apply_claude(ctx: click.Context, credential: ClaudeCredential, listed: Sequence[str], model: str | None) -> None: +def _apply_claude(ctx: click.Context, credential: ClaudeCredential, listing: _Listing, model: str | None) -> None: ctx_obj: Final[CliContextObj] = ctx.obj base_url: Final = ctx_obj["base_url"] - if model is not None and model not in listed: + listed: Final = listing.ids + starting: Final = _starting_model(model, listing) if model is not None else None + if model is not None and starting is None: shown: Final = ", ".join(listed[:_LISTED_MODELS_SHOWN]) more: Final = f", and {len(listed) - _LISTED_MODELS_SHOWN} more" if len(listed) > _LISTED_MODELS_SHOWN else "" raise click.ClickException( @@ -113,29 +138,32 @@ def _apply_claude(ctx: click.Context, credential: ClaudeCredential, listed: Sequ configure_claude_settings( base_url, credential, - _model_choice(model), + _model_choice(starting), settings_path, configure_state_path(settings_path), settings_file_owners(settings_path), ) except ClaudeSettingsError as e: raise click.ClickException(str(e)) - in_picker: Final = sum(1 for listed_model in listed if _CLAUDE_CODE_PICKER_FILTER.search(listed_model)) + in_picker: Final = sum(1 for listed_model in listed if CLAUDE_CODE_PICKER_PATTERN.search(listed_model)) click.echo(f"Configured Claude Code: {settings_path} now routes through {base_url}.") + click.echo( "Credential: your virtual key, stored in the file as ANTHROPIC_AUTH_TOKEN." if isinstance(credential, StaticToken) else "Credential: your `lite login`, read through apiKeyHelper on every request, so a later login renews it." ) click.echo( - f"Starting model: {model} ({STARTING_MODEL_ROLE}); switch any time with /model." - if model is not None + f"Starting model: {starting} ({STARTING_MODEL_ROLE}); switch any time with /model." + if starting is not None else "Starting model: not pinned (Claude Code's default, or a model you set yourself); switch with /model, or " "pass --model to start on a proxy model." ) click.echo( - f"/model will list {in_picker} of the proxy's {len(listed)} models (Claude Code shows only ids containing " - "'claude' or 'anthropic')." + f"/model will list all {len(listed)} of the proxy's models." + if in_picker == len(listed) + else f"/model will list {in_picker} of the proxy's {len(listed)} models: Claude Code shows only ids containing " + "'claude' or 'anthropic', and this proxy does not list the rest under such names." ) click.echo("Start `claude` from any terminal. Undo with `lite unconfigure claude`.") if isinstance(credential, StaticToken) and settings_path.is_symlink(): @@ -173,8 +201,10 @@ def interactive_configure( targets: Final = pick_targets() if _CLAUDE_TARGET not in targets: return - credential, listed = _start(ctx, None) - _apply_claude(ctx, credential, listed, pick_model(listed)) + credential, listing = _start(ctx, None) + _apply_claude( + ctx, credential, listing, pick_model(tuple(model.source_model or model.id for model in listing.models)) + ) @click.group(name="configure", invoke_without_command=True) @@ -218,8 +248,8 @@ def configure_claude(ctx: click.Context, api_key: str | None, model: str | None) setting is kept, and what changed is recorded so `lite unconfigure claude` can put it back. Assumes the proxy is already running. """ - credential, listed = _start(ctx, api_key) - _apply_claude(ctx, credential, listed, model) + credential, listing = _start(ctx, api_key) + _apply_claude(ctx, credential, listing, model) @unconfigure_group.command(name="claude") diff --git a/litellm/proxy/client/cli/commands/pi.py b/litellm/proxy/client/cli/commands/pi.py index 70c89a853e3..9810e81ae36 100644 --- a/litellm/proxy/client/cli/commands/pi.py +++ b/litellm/proxy/client/cli/commands/pi.py @@ -13,10 +13,11 @@ from dataclasses import dataclass from enum import StrEnum from pathlib import Path from types import MappingProxyType -from typing import Final +from typing import Annotated, Final import requests -from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator +from pydantic.types import StringConstraints PI_CONFIG_DIR_ENV: Final = "PI_CODING_AGENT_DIR" PI_PROVIDER_NAME: Final = "litellm" @@ -51,12 +52,25 @@ class ModelLimits: max_tokens: int | None -class _Model(BaseModel): - id: str +_NonEmptyString = Annotated[str, StringConstraints(min_length=1)] + + +class ListedModel(BaseModel): + model_config = ConfigDict(frozen=True) + + id: _NonEmptyString + source_model: _NonEmptyString | None = None class _ModelList(BaseModel): - data: tuple[_Model, ...] + data: tuple[ListedModel, ...] + + @model_validator(mode="after") + def unique_id_mappings(self) -> "_ModelList": + mappings: Final = frozenset((model.id, model.source_model or model.id) for model in self.data) + if len(frozenset(model.id for model in self.data)) != len(mappings): + raise ValueError("model ids must not map to multiple source models") + return self class _ModelGroup(BaseModel): @@ -69,17 +83,18 @@ class _ModelGroupList(BaseModel): data: tuple[_ModelGroup, ...] -def fetch_model_ids( +def fetch_model_listing( base_url: str, api_key: str, *, get: Callable[..., requests.Response] = requests.get, -) -> tuple[str, ...] | PiSyncError: + headers: Mapping[str, str] = MappingProxyType({}), +) -> tuple[ListedModel, ...] | PiSyncError: url: Final = base_url.rstrip("/") + "/v1/models" try: resp: Final = get( url, - headers={"Authorization": f"Bearer {api_key}"}, # mutable-ok: requests headers require a dict + headers={"Authorization": f"Bearer {api_key}", **headers}, # mutable-ok: requests headers require a dict timeout=10, ) except requests.RequestException as e: @@ -94,10 +109,21 @@ def fetch_model_ids( listing: Final = _ModelList.model_validate(resp.json()) except (ValueError, ValidationError) as e: return PiSyncError(f"Unexpected /v1/models response from the proxy: {e}", kind=ListingFailure.BAD_BODY) - ids: Final = tuple(dict.fromkeys(model.id for model in listing.data)) - if not ids: + models: Final = tuple(dict.fromkeys(listing.data)) + if not models: return PiSyncError("The proxy returned no models for your key.", kind=ListingFailure.EMPTY) - return ids + return models + + +def fetch_model_ids( + base_url: str, + api_key: str, + *, + get: Callable[..., requests.Response] = requests.get, + headers: Mapping[str, str] = MappingProxyType({}), +) -> tuple[str, ...] | PiSyncError: + listed: Final = fetch_model_listing(base_url, api_key, get=get, headers=headers) + return listed if isinstance(listed, PiSyncError) else tuple(dict.fromkeys(model.id for model in listed)) _NO_LIMITS: Final[Mapping[str, ModelLimits]] = MappingProxyType({}) @@ -222,11 +248,13 @@ __all__ = ( "LITELLM_PROXY_API_KEY_ENV", "PI_CONFIG_DIR_ENV", "PI_PROVIDER_NAME", + "ListedModel", "ListingFailure", "ModelLimits", "PiSyncError", "fetch_model_ids", "fetch_model_limits", + "fetch_model_listing", "models_json_path", "provider_block", "sync_models_json", diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 213a697b3dd..4b3ff2711b3 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -10,12 +10,24 @@ legacy internal names with `general_settings.use_team_public_model_name: false`. from __future__ import annotations -from collections.abc import Mapping, Sequence +import re +from collections.abc import Container, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast +import litellm + if TYPE_CHECKING: from litellm.router import Router + from litellm.types.proxy.model_listing import ModelInfoResponse + +CLAUDE_CODE_PICKER_PATTERN: Final = re.compile(r"claude|anthropic", re.IGNORECASE) +GATEWAY_CLIENT_HEADER: Final = "x-gateway-client" +CLAUDE_CODE_CLIENT: Final = "claude-code" +_CLAUDE_CODE_ALIAS_PREFIX: Final = "claude-router-" +_ONE_MILLION_SUFFIX: Final = "[1m]" +_ONE_MILLION_TOKENS: Final = 1_000_000 def configured_display_names( @@ -40,6 +52,115 @@ def configured_display_names( ) +def _unmarked(name: str) -> str: + return name[: -len(_ONE_MILLION_SUFFIX)] if name.lower().endswith(_ONE_MILLION_SUFFIX) else name + + +def _compatibility_id(model_id: str) -> str: + return f"{_CLAUDE_CODE_ALIAS_PREFIX}{model_id.encode().hex()}" + + +def _decoded_compatibility_id(view_id: str) -> str | None: + encoded: Final = _unmarked(view_id).removeprefix(_CLAUDE_CODE_ALIAS_PREFIX) + if encoded == _unmarked(view_id): + return None + try: + model_id: Final = bytes.fromhex(encoded).decode() + except (ValueError, UnicodeDecodeError): + return None + return model_id if _compatibility_id(model_id) == _unmarked(view_id) else None + + +def claude_code_model_id( + model_id: str, + max_input_tokens: float | None, + routing_names: Container[str], +) -> str: + """The collision-free id Claude Code's picker lists a model under.""" + if "*" in model_id: + return model_id + shaped: Final = model_id if CLAUDE_CODE_PICKER_PATTERN.search(model_id) else _compatibility_id(model_id) + one_million: Final = max_input_tokens is not None and max_input_tokens >= _ONE_MILLION_TOKENS + marked: Final = ( + f"{shaped}{_ONE_MILLION_SUFFIX}" if one_million and not shaped.lower().endswith(_ONE_MILLION_SUFFIX) else shaped + ) + return next( + ( + name + for name in (marked, shaped) + if name == model_id or claude_code_group_name(name, routing_names) == model_id + ), + model_id, + ) + + +def claude_code_group_name(view_id: str, routing_names: Container[str]) -> str | None: + """Decode a canonical compatibility id only when no configured route claims it.""" + if view_id in routing_names: + return None + unmarked: Final = _unmarked(view_id) + if unmarked != view_id and unmarked in routing_names: + return unmarked + model_id: Final = _decoded_compatibility_id(view_id) + return model_id if model_id and model_id in routing_names else None + + +def is_claude_code_client(headers: Mapping[str, str]) -> bool: + """Claude Code itself, or a client asking for its view of the listing the way Ramp Router's does""" + from litellm.llms.anthropic.common_utils import is_claude_code_user_agent + + return ( + is_claude_code_user_agent(headers.get("user-agent", "")) + or headers.get(GATEWAY_CLIENT_HEADER, "").lower() == CLAUDE_CODE_CLIENT + ) + + +def claude_code_view_ids( + rows: Sequence[ModelInfoResponse], + headers: Mapping[str, str], + routing_names: Container[str], +) -> Mapping[str, str]: + """served id -> Claude Code id for the requested listing view""" + if not is_claude_code_client(headers): + return MappingProxyType({}) + return MappingProxyType( + {row["id"]: claude_code_model_id(row["id"], row.get("max_input_tokens"), routing_names) for row in rows} + ) + + +@dataclass(frozen=True, slots=True) +class ClaudeCodeRoutingNames: + """Existing routes always own their names, including aliases and wildcard routes.""" + + llm_router: Router | None + team_id: str | None = None + alias_maps: tuple[object, ...] = () + + def __contains__(self, name: object) -> bool: + if not isinstance(name, str): + return False + if name in litellm.model_alias_map or any( + isinstance(aliases, Mapping) and name in aliases for aliases in self.alias_maps + ): + return True + if self.llm_router is None: + return False + return ( + name in self.llm_router.model_group_alias + or self.llm_router.has_model_id(name) + or bool(self.llm_router.get_candidate_model_ids_for_route(name, self.team_id)) + ) + + +def claude_code_requested_group( + requested: str, + llm_router: Router, + team_id: str | None, + alias_maps: tuple[object, ...] = (), +) -> str | None: + return claude_code_group_name(requested, ClaudeCodeRoutingNames(llm_router, team_id, alias_maps)) + + class TeamModelNameTranslator: """Translates internal team routing keys to their public names for the model listing/retrieve responses. Stateless; the live router and general_settings diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3bbe932f5d0..9269fd48e6c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -366,8 +366,11 @@ from litellm.proxy.common_utils.load_config_utils import ( ) from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( + ClaudeCodeRoutingNames, TeamModelNameTranslator, + claude_code_view_ids, configured_display_names, + is_claude_code_client, ) from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, @@ -6639,6 +6642,14 @@ class ProxyConfig: return parsed return None + async def get_hierarchical_router_settings( + self, + user_api_key_dict: UserAPIKeyAuth | None, + prisma_client: PrismaClient | None, + proxy_logging_obj: ProxyLogging | None = None, + ) -> dict | None: + return await self._get_hierarchical_router_settings(user_api_key_dict, prisma_client, proxy_logging_obj) + async def _get_hierarchical_router_settings( self, user_api_key_dict: Optional["UserAPIKeyAuth"], @@ -8632,6 +8643,7 @@ _STREAM_KEEPALIVE: Final = object() _KEEPALIVE_MIN_SECONDS: Final = 1.0 _KEEPALIVE_MAX_SECONDS: Final = 300.0 _EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({}) +_EMPTY_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) async def _iter_with_keepalive( @@ -10469,6 +10481,24 @@ async def model_list( wants_anthropic_format: Final = ( http_request is not None and http_request.headers.get("anthropic-version") is not None ) + client_headers: Final[Mapping[str, str]] = http_request.headers if http_request is not None else _EMPTY_HEADERS + view_router_settings: Final = ( + await proxy_config.get_hierarchical_router_settings(user_api_key_dict, prisma_client, proxy_logging_obj) + if wants_anthropic_format and is_claude_code_client(client_headers) + else None + ) + view_aliases: Final = ( + view_router_settings.get("model_group_alias") if isinstance(view_router_settings, Mapping) else None + ) + routing_names: Final = ClaudeCodeRoutingNames( + llm_router, + team_id or user_api_key_dict.team_id, + ( + user_api_key_dict.aliases, + user_api_key_dict.team_model_aliases, + view_aliases, + ), + ) # Validate scope parameter if provided if scope is not None and scope != "expand": @@ -10555,6 +10585,11 @@ async def model_list( return create_anthropic_model_list_response( admin_listing, display_names=configured_display_names(admin_entries, llm_router), + listed_ids=claude_code_view_ids( + admin_listing, + client_headers, + routing_names, + ), ) return dict( @@ -10603,6 +10638,11 @@ async def model_list( return create_anthropic_model_list_response( listing, display_names=configured_display_names(entries, llm_router), + listed_ids=claude_code_view_ids( + listing, + client_headers, + routing_names, + ), ) return dict( diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 91652c89092..e1b39c4ba13 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -2267,3 +2267,25 @@ def test_create_anthropic_model_list_response_empty(): assert response["has_more"] is False assert response["first_id"] is None assert response["last_id"] is None + + +def test_create_anthropic_model_list_response_lists_ids_as_told(): + """listed_ids renames an entry for the caller while display_name and every other field stay keyed to the served + id, and the envelope's first/last ids follow the renamed entries.""" + from litellm.llms.anthropic.common_utils import ( + create_anthropic_model_list_response, + ) + + response = create_anthropic_model_list_response( + [ + {"id": "gpt-4o", "object": "model", "created": 0, "owned_by": "openai", "max_input_tokens": 1000000}, + {"id": "claude-haiku-4-5", "object": "model", "created": 0, "owned_by": "openai"}, + ], + display_names={"gpt-4o": "GPT 4o"}, + listed_ids={"gpt-4o": "claude-router-gpt-4o[1m]"}, + ) + + gpt, haiku = response["data"] + assert (gpt["id"], gpt["display_name"], gpt["max_input_tokens"]) == ("claude-router-gpt-4o[1m]", "GPT 4o", 1000000) + assert (haiku["id"], haiku["display_name"]) == ("claude-haiku-4-5", "claude-haiku-4-5") + assert (response["first_id"], response["last_id"]) == ("claude-router-gpt-4o[1m]", "claude-haiku-4-5") diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 8e6c41761cc..6cce6d0316b 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -7249,3 +7249,46 @@ async def test_jwt_builder_returns_every_team_grant_the_key_path_gets(is_proxy_a assert token.team_member == Member(user_id="jwt-user", role="admin") assert token.team_member_spend == 1.5 assert token.jwt_claims == {"sub": "jwt-user"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/v1/messages", "/messages", "/v1/chat/completions", "/chat/completions", "/v1/responses", "/responses"]) +async def test_claude_view_normalizes_before_model_access(monkeypatch, route): + from starlette.requests import Request + from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access + + source = "foo[1m]" + encoded = "claude-router-" + source.encode().hex() + "[1m]" + router = litellm.Router(model_list=[{"model_name": source, "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}}]) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + data = {"model": encoded, "messages": [{"role": "user", "content": "hi"}]} + request = Request({"type": "http", "method": "POST", "path": route, "headers": [], "query_string": b""}) + token = UserAPIKeyAuth(models=[source]) + await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route=route, request=request, llm_model_list=router.model_list, llm_router=router) + assert data["model"] == source + assert (await request.json())["model"] == source + assert json.loads(await request.body())["model"] == source + assert request.scope["parsed_body"][1]["model"] == source + with pytest.raises(ProxyException): + await _enforce_key_and_fallback_model_access(valid_token=UserAPIKeyAuth(models=["other"]), request_data=data, route=route, request=request, llm_model_list=router.model_list, llm_router=router) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("layer", ["literal", "global", "router", "key", "hierarchical", "unclaimed"]) +async def test_claude_view_never_reinterprets_explicit_names(monkeypatch, layer): + from starlette.requests import Request + from litellm.proxy.auth.user_api_key_auth import _normalize_claude_model + + encoded = "claude-router-666f6f" + names = ("foo", "other", encoded) if layer == "literal" else ("foo", "other") + alias = {encoded: "other"} + router = litellm.Router(model_list=[{"model_name": name, "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}} for name in names], model_group_alias=alias if layer == "router" else None) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "model_alias_map", alias if layer == "global" else {}) + token = UserAPIKeyAuth(aliases=alias if layer == "key" else {}, router_settings={"model_group_alias": alias} if layer == "hierarchical" else None) + data = {"model": encoded} + request = Request({"type": "http", "method": "POST", "path": "/v1/messages", "headers": [], "query_string": b""}) + await _normalize_claude_model(data, token, request, "/v1/messages") + assert data["model"] == ("foo" if layer == "unclaimed" else encoded) + await _normalize_claude_model(data, token, request, "/v1/messages") + assert data["model"] == ("foo" if layer == "unclaimed" else encoded) diff --git a/tests/test_litellm/proxy/client/cli/test_configure_commands.py b/tests/test_litellm/proxy/client/cli/test_configure_commands.py index a1808383982..ac7408339fe 100644 --- a/tests/test_litellm/proxy/client/cli/test_configure_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_configure_commands.py @@ -89,7 +89,7 @@ class TestConfigureClaudeWithAVirtualKey: assert "Starting model: claude-auto" in result.output assert "1 of the proxy's 2 models" in result.output assert "lite unconfigure claude" in result.output - assert len(responses.calls) == 1 + assert [call.request.headers.get("x-gateway-client") for call in responses.calls] == ["claude-code"] @responses.activate def test_takes_the_key_from_the_global_option_and_keeps_claude_codes_default(self, runner, paths): @@ -358,3 +358,74 @@ class TestUnconfigureClaude: result = runner.invoke(cli, ["unconfigure", "claude"]) assert result.exit_code != 0 assert "nothing to undo" in result.output + + +class TestClaudeCodeView: + VIEW = {"anthropic-version": "2023-06-01", "x-gateway-client": "claude-code"} + + def _mock(self, rows): + responses.get( + f"{PROXY}/v1/models", + json={"data": rows}, + match=[responses.matchers.header_matcher({"Authorization": f"Bearer {VALID_KEY}", **self.VIEW})], + ) + + @responses.activate + @pytest.mark.parametrize( + "model, pinned", + [ + ("literal-claude-router-source", "emitted-literal"), + ("marked-sibling", "emitted-marked[1m]"), + ("emitted-collision", "emitted-source-priority"), + ("emitted-only", "emitted-only"), + ], + ) + def test_pins_source_identity_before_emitted_id(self, runner, paths, model, pinned): + self._mock( + [ + {"id": "emitted-collision", "source_model": "other-source"}, + {"id": "emitted-source-priority", "source_model": "emitted-collision"}, + {"id": "emitted-marked[1m]", "source_model": "marked-sibling"}, + {"id": "emitted-literal", "source_model": "literal-claude-router-source"}, + {"id": "emitted-only"}, + ] + ) + settings_path, _ = paths + result = _configure(runner, "--api-key", VALID_KEY, "--model", model) + assert result.exit_code == 0, result.output + assert json.loads(settings_path.read_text())["model"] == pinned + assert f"Starting model: {pinned}" in result.output + assert len(responses.calls) == 1 + + @responses.activate + def test_refuses_unknown_short_suffix(self, runner, paths): + self._mock([{"id": "emitted-router-source", "source_model": "literal-router-source"}]) + settings_path, _ = paths + result = _configure(runner, "--api-key", VALID_KEY, "--model", "source") + assert result.exit_code != 0 + assert "'source' is not served" in result.output + assert not settings_path.exists() + + @responses.activate + def test_interactive_picker_uses_source_names(self, paths): + self._mock([{"id": "emitted", "source_model": "source"}]) + settings_path, _ = paths + asked = {} + ctx = click.Context( + configure_group, obj={"base_url": PROXY, "api_key": VALID_KEY, "api_key_from_token_file": False} + ) + + def pick_model(listed): + asked["listed"] = tuple(listed) + return "source" + + interactive_configure(ctx, pick_targets=lambda: ("claude",), pick_model=pick_model) + assert asked["listed"] == ("source",) + assert json.loads(settings_path.read_text())["model"] == "emitted" + + @responses.activate + def test_counts_what_an_older_proxy_lets_the_picker_show(self, runner, paths): + _mock_models() + result = _configure(runner, "--api-key", VALID_KEY) + assert result.exit_code == 0, result.output + assert "/model will list 1 of the proxy's 2 models: Claude Code shows only ids containing" in result.output diff --git a/tests/test_litellm/proxy/client/cli/test_pi.py b/tests/test_litellm/proxy/client/cli/test_pi.py index 1c2da514ee2..03e5d7dd197 100644 --- a/tests/test_litellm/proxy/client/cli/test_pi.py +++ b/tests/test_litellm/proxy/client/cli/test_pi.py @@ -13,6 +13,7 @@ from litellm.proxy.client.cli.commands.pi import ( PiSyncError, fetch_model_ids, fetch_model_limits, + fetch_model_listing, models_json_path, provider_block, sync_models_json, @@ -50,6 +51,43 @@ class TestFetchModelIds: assert captured["url"] == "http://localhost:4000/v1/models" assert captured["headers"] == {"Authorization": "Bearer sk-key"} + def test_returns_rows_with_optional_source_model_and_dedups_identical_rows(self): + result = fetch_model_listing( + "http://localhost:4000", + "sk-key", + get=lambda *a, **k: _FakeResponse( + 200, + {"data": [{"id": "emitted", "source_model": "source"}, {"id": "emitted", "source_model": "source"}]}, + ), + ) + assert not isinstance(result, PiSyncError) + assert tuple((model.id, model.source_model) for model in result) == (("emitted", "source"),) + + @pytest.mark.parametrize( + "entry", + [ + {"id": ""}, + {"id": "emitted", "source_model": ""}, + {"id": "emitted", "source_model": 1}, + ], + ) + def test_rejects_invalid_model_identity(self, entry): + result = fetch_model_listing( + "http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(200, {"data": [entry]}) + ) + assert isinstance(result, PiSyncError) and result.kind is ListingFailure.BAD_BODY + + def test_rejects_conflicting_emitted_id_mappings(self): + result = fetch_model_listing( + "http://localhost:4000", + "sk-key", + get=lambda *a, **k: _FakeResponse( + 200, + {"data": [{"id": "emitted", "source_model": "one"}, {"id": "emitted", "source_model": "two"}]}, + ), + ) + assert isinstance(result, PiSyncError) and result.kind is ListingFailure.BAD_BODY + def test_network_error_is_a_value(self): def boom(*a, **k): raise requests.ConnectionError("refused") diff --git a/tests/test_litellm/proxy/common_utils/test_model_listing_utils.py b/tests/test_litellm/proxy/common_utils/test_model_listing_utils.py new file mode 100644 index 00000000000..7ef03140093 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_model_listing_utils.py @@ -0,0 +1,109 @@ +"""Model identity survives Claude Code presentation, filtering and configured alias precedence.""" + +from itertools import combinations + +import pytest + +from litellm import Router +from litellm.proxy.common_utils.model_listing_utils import ( + ClaudeCodeRoutingNames, + claude_code_group_name, + claude_code_model_id, + claude_code_requested_group, + claude_code_view_ids, +) + + +def _encoded(name): + return "claude-router-" + name.encode().hex() + + +def _marked(name): + return f"{_encoded(name)}[1m]" + + +def _row(name, limit=1000000): + return {"id": name, "object": "model", "created": 0, "owned_by": "openai", "max_input_tokens": limit} + + +def _router(*names, aliases=None): + return Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}} + for name in names + ], + model_group_alias=aliases, + ) + + +@pytest.mark.parametrize("limit", [None, 999999, 1000000]) +@pytest.mark.parametrize("name", ["foo", "foo[1m]", "foo[1M]", "a/b: 世界", "claude-router-foo", "claude-opus-5", "claude-opus-5[1m]"]) +def test_listing_round_trips_entire_source_name(name, limit): + names = frozenset({name}) + view = claude_code_model_id(name, limit, names) + assert (claude_code_group_name(view, names) or view) == name + if "claude" not in name: + assert view.startswith(_encoded(name)) + assert ("[1m]" in view.lower()) == (limit == 1000000 or "[1m]" in name.lower() and "claude" in name) + + +def test_collision_matrix_round_trips_without_duplicate_ids(): + universe = ("foo", "foo[1m]", "claude-router-foo", _encoded("foo"), _encoded("foo") + "[1m]", "claude-opus-5", "claude-opus-5[1m]") + for pair in combinations(universe, 2): + for visible in (pair, pair[:1], pair[1:]): + names = frozenset(pair) + view = claude_code_view_ids(tuple(_row(n) for n in visible), {"user-agent": "claude-code/2.1.267"}, names) + assert len(set(view.values())) == len(visible) + assert all((claude_code_group_name(shown, names) or shown) == source for source, shown in view.items()) + + +@pytest.mark.parametrize("spelling", ["claude-router-foo", "claude-router-ff", "claude-router-66 6f6f", "claude-router-666F6F", "claude-router-", _encoded("missing")]) +def test_unknown_or_noncanonical_ids_are_never_guessed(spelling): + assert claude_code_group_name(spelling, frozenset({"foo"})) is None + + +@pytest.mark.parametrize("headers,enabled", [ + ({"user-agent": "claude-code/2.1.267"}, True), + ({"user-agent": "claude-cli/2.1.267 (external, sdk-cli)"}, True), + ({"x-gateway-client": "Claude-Code"}, True), + ({"user-agent": "anthropic-sdk-python/0.40"}, False), + ({}, False), +]) +def test_only_claude_code_gets_the_view(headers, enabled): + rows = (_row("foo"), _row("claude-opus-5")) + view = claude_code_view_ids(rows, headers, frozenset(row["id"] for row in rows)) + assert dict(view) == ({"foo": _encoded("foo") + "[1m]", "claude-opus-5": "claude-opus-5[1m]"} if enabled else {}) + + +@pytest.mark.parametrize("layer", ["literal", "global", "router", "key", "team", "wildcard"]) +def test_configured_names_outrank_generated_ids_even_when_hidden_from_listing(monkeypatch, layer): + import litellm + + encoded = _encoded("foo") + alias = {encoded: "other"} + monkeypatch.setattr(litellm, "model_alias_map", alias if layer == "global" else {}) + router = _router("foo", "other", *( (encoded,) if layer == "literal" else ("*",) if layer == "wildcard" else ()), aliases=alias if layer == "router" else None) + maps = (alias,) if layer in ("key", "team") else () + names = ClaudeCodeRoutingNames(router, None, maps) + assert claude_code_requested_group(encoded, router, None, maps) is None + assert claude_code_view_ids((_row("foo", None),), {"x-gateway-client": "claude-code"}, names)["foo"] == "foo" + + +@pytest.mark.parametrize("source", ["foo", "foo[1m]", "世界"]) +def test_mutation_breaking_the_hex_name_cannot_route_to_the_source(source): + router = _router(source) + encoded = _encoded(source) + malformed = encoded[:-1] + ("0" if encoded[-1] != "0" else "1") + assert claude_code_requested_group(malformed, router, None) is None + assert claude_code_requested_group(_marked(source), router, None) == source + + +def test_team_public_name_uses_the_same_scope_at_list_and_request(): + router = Router(model_list=[{ + "model_name": "model_name_team-a_id", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "shared"}, + }]) + shown = claude_code_view_ids((_row("shared"),), {"user-agent": "claude-code/2.1.267"}, ClaudeCodeRoutingNames(router, "team-a"))["shared"] + assert claude_code_requested_group(shown, router, "team-a") == "shared" + assert claude_code_requested_group(shown, router, "team-b") is None diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index bc6106a06f8..e3d6b46e2d2 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -343,3 +343,53 @@ def test_anthropic_format_returns_public_team_model_name( assert response.status_code == 200 assert [m["id"] for m in response.json()["data"]] == ["gpt-4-team"] assert internal_name not in response.text + + +@pytest.mark.parametrize("path", ["/v1/models", "/models"]) +@pytest.mark.parametrize( + "caller_headers", + [ + {"anthropic-version": "2023-06-01", "user-agent": "claude-code/2.1.267"}, + {"anthropic-version": "2023-06-01", "user-agent": "claude-cli/2.1.267 (external, sdk-cli)"}, + {"anthropic-version": "2023-06-01", "x-gateway-client": "claude-code"}, + ], +) +def test_anthropic_format_lists_claude_code_view_ids_for_claude_code( + client, auth_as, patched_models, monkeypatch, path, caller_headers +): + """Claude Code drops every id without claude/anthropic in it and reads [1m] as its 1M marker, so for Claude + Code (its discovery fetch's own user agent, its SDK's, or the gateway-client header a launcher sends) every + group is listed under a Claude-shaped id with the marker where the window reaches 1M; the display name stays + the served name.""" + + def _create_model_info_response(model_id, provider="openai", **kwargs): + if model_id != "claude-sonnet": + return _stub_model_info_response(model_id=model_id, provider=provider) + return {**_stub_model_info_response(model_id=model_id, provider=provider), "max_input_tokens": 1000000} + + patched_models.model_group_alias = {} + patched_models.has_model_id.return_value = False + patched_models.get_candidate_model_ids_for_route.side_effect = lambda name, team_id=None: frozenset({name}) if name in ("gpt-4", "claude-sonnet") else frozenset() + monkeypatch.setattr(proxy_utils, "create_model_info_response", _create_model_info_response) + + with auth_as(): + response = client.get(path, headers=caller_headers) + + assert response.status_code == 200 + body = response.json() + assert [(m["id"], m["display_name"]) for m in body["data"]] == [ + ("claude-router-6770742d34", "gpt-4"), + ("claude-sonnet[1m]", "claude-sonnet"), + ] + assert (body["first_id"], body["last_id"]) == ("claude-router-6770742d34", "claude-sonnet[1m]") + assert [row["source_model"] for row in body["data"]] == ["gpt-4", "claude-sonnet"] + + +@pytest.mark.parametrize("path", ["/v1/models", "/models"]) +def test_anthropic_format_keeps_served_ids_for_other_anthropic_clients(client, auth_as, patched_models, path): + """An Anthropic SDK asking for the vendor shape gets the served ids: the view is Claude Code's alone.""" + with auth_as(): + response = client.get(path, headers={"anthropic-version": "2023-06-01", "user-agent": "anthropic-sdk-python/0.40"}) + + assert response.status_code == 200 + assert [m["id"] for m in response.json()["data"]] == ["gpt-4", "claude-sonnet"] From 13837d319daf1a38524a0c08ab929aef1ebc0285 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 9 Sep 2026 22:08:30 -0700 Subject: [PATCH 297/411] fix(openai): drop tool schema regex patterns OpenAI's validator cannot compile (#40485) * fix(openai): drop tool schema regex patterns OpenAI's validator cannot compile OpenAI validates function tool parameters with jsonschema's format checker, which compiles every pattern with Python re. Claude Code's Artifact tool ships an ECMA-262 pattern with \p{..} Unicode property escapes, so any OpenAI target behind /v1/messages, /v1/responses or /v1/chat/completions 400s with "Invalid schema for function 'Artifact': '...' is not a 'regex'" for every model family. Drop only the patterns Python re rejects, keep the rest, at the same seams that already flatten top-level combinators. * fix(openai): walk only schema positions, iteratively, and drop regexes for every openai deployment Review round: the regex sanitizer now walks JSON Schema applicator positions only (properties, items, prefixItems, combinators, $defs, additionalProperties and the rest), so a pattern key inside default, examples, const or a vendor extension is data and stays. It also drops patternProperties keys Python re cannot compile, which OpenAI checks the same way. The walk is level-order and rebuilt deepest level first instead of recursive, so the code-quality recursion gate passes and there is no depth cap below what a JSON parser admits. On the chat wire an openai deployment with a custom api_base now drops such regexes too, since that base is usually a proxy in front of the same validator, while the lossier combinator flattening stays limited to api.openai.com hosts. --- .../prompt_templates/common_utils.py | 123 +++++++++++++- litellm/llms/azure/chat/gpt_transformation.py | 16 +- .../azure/chat/o_series_transformation.py | 4 +- .../llms/openai/chat/gpt_transformation.py | 36 ++-- .../llms/openai/responses/transformation.py | 89 +++++----- ...ore_utils_prompt_templates_common_utils.py | 154 +++++++++++++++++- .../test_azure_chat_gpt_transformation.py | 23 +++ .../response/test_azure_transformation.py | 29 ++++ .../chat/test_openai_gpt_transformation.py | 51 ++++++ .../test_openai_responses_transformation.py | 83 ++++++++++ 10 files changed, 532 insertions(+), 76 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index d70530534da..2485896184e 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,8 +6,8 @@ import io import json import mimetypes import re -from collections.abc import Iterable, Iterator, Mapping, Sequence -from itertools import groupby +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from itertools import groupby, islice from os import PathLike from pathlib import Path from types import MappingProxyType @@ -1320,17 +1320,128 @@ def flatten_top_level_schema_combinators(schema: Mapping[str, object]) -> Mappin return _flatten_schema_against_root(schema, schema, frozenset(), 0, {}) # mutable-ok: fresh per-call $ref memo -def tool_with_flattened_parameters(tool: Mapping[str, object]) -> Mapping[str, object]: +_SUBSCHEMA_KEYWORDS: Final = frozenset( + { + "additionalItems", + "additionalProperties", + "contains", + "else", + "if", + "items", + "not", + "propertyNames", + "then", + "unevaluatedItems", + "unevaluatedProperties", + } +) +_SUBSCHEMA_LIST_KEYWORDS: Final = frozenset({"allOf", "anyOf", "items", "oneOf", "prefixItems"}) +_SUBSCHEMA_MAP_KEYWORDS: Final = frozenset( + {"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"} +) + +_MAX_SCHEMA_NESTING: Final = 1024 + + +def drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: + """Drop every regex in a schema position that Python's ``re`` cannot compile. + + OpenAI validates tool ``parameters`` against the 2020-12 metaschema with + ``jsonschema``'s format checker, which hands each ``pattern`` value and each + ``patternProperties`` key to ``re.compile``, so a regex written for an + ECMA-262 engine (Unicode property escapes such as ``\\p{Cc}``, as in Claude + Code's ``Artifact`` tool) is refused with "'...' is not a 'regex'" by every + model family on both the chat and Responses wires. Only schema positions are + walked (properties, items, combinators, ``$defs`` and the other applicators), + so a ``pattern`` key inside ``default``, ``examples``, ``const`` or vendor + extensions is data and stays. Outside strict mode the keyword is only a + hint, so dropping it costs the model a constraint and the caller nothing. + Compilable regexes and everything else pass through, the input is never + mutated, and the same object comes back when nothing was dropped. The walk + is level-order rather than recursive, rebuilt deepest level first, and stops + at more schema levels than a JSON parser admits, so a cyclic schema built in + code cannot spin it. + """ + rebuilt: dict[int, Mapping[str, object]] = {} # mutable-ok: per-call memo of rewritten nodes, deepest level first + for level in reversed(tuple(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))): + rebuilt.update( + (id(node), rewritten) + for node in level + if (rewritten := _node_without_non_python_regex(node, rebuilt)) is not node + ) + return rebuilt.get(id(schema), schema) + + +def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str, object], ...]]: + frontier: tuple[Mapping[str, object], ...] = (schema,) # rebind-ok: level-order cursor, one level a round + while frontier: + yield frontier + frontier = tuple(child for node in frontier for child in _subschemas(node)) + + +def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]: + for key, value in node.items(): + if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): + yield from (sub for sub in value.values() if isinstance(sub, dict)) + elif key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): + yield from (sub for sub in value if isinstance(sub, dict)) + elif key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): + yield value + + +def _node_without_non_python_regex( + node: Mapping[str, object], rebuilt: Mapping[int, Mapping[str, object]] +) -> Mapping[str, object]: + kept: Final = { # mutable-ok: tool parameters are JSON dicts + key: _keyword_value_rebuilt(key, value, rebuilt) + for key, value in node.items() + if key != "pattern" or not isinstance(value, str) or _is_python_regex(value) + } + return node if len(kept) == len(node) and all(kept[key] is node[key] for key in kept) else kept + + +def _keyword_value_rebuilt(key: str, value: object, rebuilt: Mapping[int, Mapping[str, object]]) -> object: + if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): + kept: Final = { # mutable-ok: tool parameters are JSON dicts + name: rebuilt.get(id(sub), sub) + for name, sub in value.items() + if key != "patternProperties" or not isinstance(name, str) or _is_python_regex(name) + } + return value if len(kept) == len(value) and all(kept[name] is value[name] for name in kept) else kept + if key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): + items: Final = [rebuilt.get(id(sub), sub) for sub in value] # mutable-ok: tool parameters are JSON lists + return value if all(new is old for new, old in zip(items, value, strict=True)) else items + if key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): + return rebuilt.get(id(value), value) + return value + + +def _is_python_regex(pattern: str) -> bool: + try: + re.compile(pattern) + except (re.error, RecursionError): + return False + return True + + +def flatten_combinators_and_drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: + return flatten_top_level_schema_combinators(drop_non_python_regex_patterns(schema)) + + +def tool_with_sanitized_parameters( + tool: Mapping[str, object], + sanitize: Callable[[Mapping[str, object]], Mapping[str, object]], +) -> Mapping[str, object]: function: Final = tool.get("function") if not isinstance(function, dict): return tool parameters: Final = function.get("parameters") if not isinstance(parameters, dict): return tool - flattened: Final = flatten_top_level_schema_combinators(parameters) - if flattened is parameters: + sanitized: Final = sanitize(parameters) + if sanitized is parameters: return tool - return {**tool, "function": {**function, "parameters": flattened}} # mutable-ok: request tools are JSON dicts + return {**tool, "function": {**function, "parameters": sanitized}} # mutable-ok: request tools are JSON dicts def _get_image_mime_type_from_url(url: str) -> str | None: diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 880a51eb584..ed16d7f3de0 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -7,8 +7,9 @@ from httpx._models import Headers, Response import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, + flatten_combinators_and_drop_non_python_regex_patterns, hoist_images_from_tool_messages, - tool_with_flattened_parameters, + tool_with_sanitized_parameters, ) from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, @@ -39,14 +40,17 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) -def flattened_tools_update(optional_params: Mapping[str, object]) -> Mapping[str, object]: +def sanitized_tools_update(optional_params: Mapping[str, object]) -> Mapping[str, object]: tools: Final = optional_params.get("tools") if not isinstance(tools, list): return _NO_TOOLS_UPDATE - flattened: Final = [ # mutable-ok: request tools are a JSON list - tool_with_flattened_parameters(tool) if isinstance(tool, dict) else tool for tool in tools + sanitized: Final = [ # mutable-ok: request tools are a JSON list + tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) + if isinstance(tool, dict) + else tool + for tool in tools ] - return MappingProxyType({"tools": flattened}) + return MappingProxyType({"tools": sanitized}) class AzureOpenAIConfig(BaseConfig): @@ -278,7 +282,7 @@ class AzureOpenAIConfig(BaseConfig): "model": model, "messages": azure_messages, **optional_params, - **flattened_tools_update(optional_params), + **sanitized_tools_update(optional_params), } def transform_response( diff --git a/litellm/llms/azure/chat/o_series_transformation.py b/litellm/llms/azure/chat/o_series_transformation.py index 246bf69cb5f..09d8075e857 100644 --- a/litellm/llms/azure/chat/o_series_transformation.py +++ b/litellm/llms/azure/chat/o_series_transformation.py @@ -20,7 +20,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.utils import get_model_info, supports_reasoning from ...openai.chat.o_series_transformation import OpenAIOSeriesConfig -from .gpt_transformation import flattened_tools_update +from .gpt_transformation import sanitized_tools_update class AzureOpenAIO1Config(OpenAIOSeriesConfig): @@ -111,6 +111,6 @@ class AzureOpenAIO1Config(OpenAIOSeriesConfig): model = model.replace("o_series/", "") # handle o_series/my-random-deployment-name flattened_params: Final = { # mutable-ok: transform_request's contract takes a plain JSON params dict **optional_params, - **flattened_tools_update(optional_params), + **sanitized_tools_update(optional_params), } return super().transform_request(model, messages, flattened_params, litellm_params, headers) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 9afc6331d96..9b410cf073e 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -19,10 +19,12 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo _should_convert_tool_call_to_json_mode, ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( + drop_non_python_regex_patterns, drop_tool_reference_parts_from_tool_messages, + flatten_combinators_and_drop_non_python_regex_patterns, get_tool_call_names, hoist_images_from_tool_messages, - tool_with_flattened_parameters, + tool_with_sanitized_parameters, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, @@ -432,7 +434,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): custom_llm_provider, api_base ) - def _flattened_tools_update_for_openai( + def _sanitized_tools_update_for_openai( self, optional_params: Mapping[str, object], litellm_params: Mapping[str, object], @@ -440,22 +442,26 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): """ OpenAI's chat completions validator rejects tool `parameters` carrying 'oneOf'/'anyOf'/'allOf'/'enum'/'const'/'not' at the top level for every - model family, unlike the Responses API, where GPT-5+ accepts them. + model family, unlike the Responses API, where GPT-5+ accepts them, and + a `pattern` Python's `re` cannot compile for every model family on both. + A custom api_base on the `openai` provider is usually a proxy in front of + the same validator, so regexes are dropped there too, while the lossier + combinator flattening stays limited to api.openai.com hosts. """ tools: Final = optional_params.get("tools") - if not isinstance(tools, list): - return _NO_TOOLS_UPDATE provider: Final = litellm_params.get("custom_llm_provider") - raw_api_base: Final = litellm_params.get("api_base") - if not self._targets_openai_hosted_endpoint( - provider if isinstance(provider, str) else None, - raw_api_base if isinstance(raw_api_base, str) else None, - ): + if not isinstance(tools, list) or provider != "openai": return _NO_TOOLS_UPDATE - flattened: Final = [ # mutable-ok: request tools are a JSON list - tool_with_flattened_parameters(tool) if isinstance(tool, dict) else tool for tool in tools + raw_api_base: Final = litellm_params.get("api_base") + sanitize: Final = ( + flatten_combinators_and_drop_non_python_regex_patterns + if self._targets_openai_hosted_endpoint(provider, raw_api_base if isinstance(raw_api_base, str) else None) + else drop_non_python_regex_patterns + ) + sanitized: Final = [ # mutable-ok: request tools are a JSON list + tool_with_sanitized_parameters(tool, sanitize) if isinstance(tool, dict) else tool for tool in tools ] - return MappingProxyType({"tools": flattened}) + return MappingProxyType({"tools": sanitized}) def transform_request( self, @@ -489,7 +495,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): "model": model, "messages": messages, **optional_params, - **self._flattened_tools_update_for_openai(optional_params, litellm_params), + **self._sanitized_tools_update_for_openai(optional_params, litellm_params), } async def async_transform_request( @@ -521,7 +527,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): "model": model, "messages": transformed_messages, **optional_params, - **self._flattened_tools_update_for_openai(optional_params, litellm_params), + **self._sanitized_tools_update_for_openai(optional_params, litellm_params), } else: ## allow for any object specific behaviour to be handled diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 97e7bcc60b7..666e9b32011 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from functools import lru_cache from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints @@ -15,6 +15,10 @@ from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( _safe_convert_created_field, ) +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + drop_non_python_regex_patterns, + flatten_combinators_and_drop_non_python_regex_patterns, +) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig @@ -40,7 +44,7 @@ else: _NO_TOOL_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) _MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("gpt-4", "gpt-3.5", "chatgpt-4o", "o1", "o3", "o4") -_PROVIDERS_WITH_COMBINATOR_REJECTING_VALIDATOR: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI}) +_PROVIDERS_WITH_OPENAI_SCHEMA_VALIDATOR: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI}) _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI}) @@ -293,7 +297,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): model=model, input=validated_input, tools=tools ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) - sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai( + sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params ) return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools @@ -378,35 +382,35 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return item return {key: value for key, value in item.items() if key != "id"} # mutable-ok: outgoing JSON request item - def _flatten_tool_schema_combinators_for_openai( + def _sanitized_tool_schemas_for_openai( self, model: str, tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, litellm_params: GenericLiteLLMParams, ) -> Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None: - """Flatten top-level schema combinators only where OpenAI's validator rejects them. + """Rewrite tool schemas only where OpenAI's validator rejects them. - OpenAI-compatible backends reusing this config (and the ChatGPT backend - Codex talks to natively) accept them, and so do GPT-5 and later models, - which also call tools better with the union intact. Codex wraps MCP tools - inside namespace entries, so nested ``tools`` arrays are walked too. - Azure OpenAI shares the validator but names deployments arbitrarily, so - the router's declared ``model_info.base_model`` wins over the deployment - name and an unrecognized name without one is left untouched. + Every model family refuses a ``pattern`` Python's ``re`` cannot compile, + while top-level schema combinators are flattened only for the families + whose validator rejects them: OpenAI-compatible backends reusing this + config (and the ChatGPT backend Codex talks to natively) accept them, + and so do GPT-5 and later models, which also call tools better with the + union intact. Codex wraps MCP tools inside namespace entries, so nested + ``tools`` arrays are walked too. Azure OpenAI shares the validator but + names deployments arbitrarily, so the router's declared + ``model_info.base_model`` wins over the deployment name and an + unrecognized name without one keeps its combinators. """ - if tools is None or self.custom_llm_provider not in _PROVIDERS_WITH_COMBINATOR_REJECTING_VALIDATOR: + if tools is None or self.custom_llm_provider not in _PROVIDERS_WITH_OPENAI_SCHEMA_VALIDATOR: return tools gate_model: Final = self._combinator_gate_model(model=model, litellm_params=litellm_params) - if not self._rejects_top_level_schema_combinators(gate_model): - return tools - flattened: Final = [ # mutable-ok: request tools are a JSON list - self._flattened_tool_or_passthrough(tool) for tool in tools - ] - return cast("Sequence[ALL_RESPONSES_API_TOOL_PARAMS]", flattened) # cast-ok: spread keeps each tool's shape - - @staticmethod - def _flattened_tool_or_passthrough(tool: object) -> object: - return OpenAIResponsesAPIConfig._flattened_tool_entry(tool) if isinstance(tool, dict) else tool + sanitize: Final = ( + flatten_combinators_and_drop_non_python_regex_patterns + if self._rejects_top_level_schema_combinators(gate_model) + else drop_non_python_regex_patterns + ) + sanitized: Final = self._sanitized_tools(tools, sanitize) + return cast("Sequence[ALL_RESPONSES_API_TOOL_PARAMS]", sanitized) # cast-ok: spread keeps each tool's shape @staticmethod def _rejects_top_level_schema_combinators(model: str) -> bool: @@ -421,35 +425,42 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return base_model if isinstance(base_model, str) and base_model else model @staticmethod - def _flattened_tool_entry( + def _sanitized_tool_entry( entry: Mapping[str, object], - ) -> dict[str, object]: # mutable-ok: request tools are JSON dicts - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - + sanitize: Callable[[Mapping[str, object]], Mapping[str, object]], + ) -> Mapping[str, object]: parameters: Final = entry.get("parameters") nested_tools: Final = entry.get("tools") + sanitized_parameters: Final = sanitize(parameters) if isinstance(parameters, dict) else parameters + sanitized_nested_tools: Final = ( + OpenAIResponsesAPIConfig._sanitized_tools(nested_tools, sanitize) + if isinstance(nested_tools, list) + else nested_tools + ) parameters_update: Final = ( - MappingProxyType({"parameters": flatten_top_level_schema_combinators(parameters)}) - if isinstance(parameters, dict) + MappingProxyType({"parameters": sanitized_parameters}) + if sanitized_parameters is not parameters else _NO_TOOL_UPDATE ) tools_update: Final = ( - MappingProxyType({"tools": OpenAIResponsesAPIConfig._flattened_nested_tools(nested_tools)}) - if isinstance(nested_tools, list) + MappingProxyType({"tools": sanitized_nested_tools}) + if sanitized_nested_tools is not nested_tools else _NO_TOOL_UPDATE ) + if not parameters_update and not tools_update: + return entry return {**entry, **parameters_update, **tools_update} # mutable-ok: request tools are JSON dicts @staticmethod - def _flattened_nested_tools( - nested_tools: Sequence[object], - ) -> list[object]: # mutable-ok: namespace tools are a JSON list - return [ # mutable-ok: namespace tools are a JSON list - OpenAIResponsesAPIConfig._flattened_tool_entry(item) if isinstance(item, dict) else item - for item in nested_tools + def _sanitized_tools( + tools: Sequence[object], + sanitize: Callable[[Mapping[str, object]], Mapping[str, object]], + ) -> Sequence[object]: + sanitized: Final = [ # mutable-ok: request tools are a JSON list + OpenAIResponsesAPIConfig._sanitized_tool_entry(item, sanitize) if isinstance(item, dict) else item + for item in tools ] + return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam: """ diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 7c1445d79c9..b5890d1a5b0 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -2,11 +2,12 @@ import copy import functools import json import os +import sys +from typing import Final from unittest.mock import MagicMock, patch import pytest - from litellm.litellm_core_utils.prompt_templates.common_utils import ( ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, @@ -25,6 +26,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( update_messages_with_model_file_ids, ) +_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' + def test_get_format_from_file_id(): unified_file_id = "litellm_proxy:application/pdf;unified_id,cbbe3534-8bf8-4386-af00-f5f6b7e370bf" @@ -1442,7 +1445,7 @@ class TestFlattenTopLevelSchemaCombinators: assert schema == snapshot -class TestToolWithFlattenedParameters: +class TestToolWithSanitizedParameters: def _anyof_tool(self): return { "type": "function", @@ -1469,11 +1472,12 @@ class TestToolWithFlattenedParameters: def test_flattens_anyof_parameters_into_new_tool(self): from litellm.litellm_core_utils.prompt_templates.common_utils import ( - tool_with_flattened_parameters, + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, ) tool = self._anyof_tool() - result = tool_with_flattened_parameters(tool) + result = tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) assert result is not tool parameters = result["function"]["parameters"] @@ -1486,7 +1490,8 @@ class TestToolWithFlattenedParameters: def test_clean_parameters_return_the_same_tool_object(self): from litellm.litellm_core_utils.prompt_templates.common_utils import ( - tool_with_flattened_parameters, + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, ) tool = { @@ -1497,7 +1502,23 @@ class TestToolWithFlattenedParameters: }, } - assert tool_with_flattened_parameters(tool) is tool + assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool + + def test_pattern_only_sanitizer_drops_the_regex_and_keeps_the_union(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + drop_non_python_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = self._anyof_tool() + tool["function"]["parameters"]["properties"]["id"]["pattern"] = _ARTIFACT_FIELD_PATTERN + + result = tool_with_sanitized_parameters(tool, drop_non_python_regex_patterns) + + parameters = result["function"]["parameters"] + assert parameters["properties"]["id"] == {"type": "string"} + assert parameters["anyOf"] == self._anyof_tool()["function"]["parameters"]["anyOf"] + assert tool["function"]["parameters"]["properties"]["id"]["pattern"] == _ARTIFACT_FIELD_PATTERN @pytest.mark.parametrize( "tool", @@ -1510,10 +1531,127 @@ class TestToolWithFlattenedParameters: ) def test_non_dict_function_or_parameters_return_the_same_tool_object(self, tool): from litellm.litellm_core_utils.prompt_templates.common_utils import ( - tool_with_flattened_parameters, + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, ) - assert tool_with_flattened_parameters(tool) is tool + assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool + + +class TestDropNonPythonRegexPatterns: + """Claude Code's Artifact tool declares ECMA-262 ``\\p{..}`` escapes that OpenAI's + validator, which compiles ``pattern`` values and ``patternProperties`` keys with + Python ``re``, refuses as "not a 'regex'".""" + + def _schema(self, pattern): + return { + "type": "object", + "properties": { + "field": {"type": "string", "pattern": pattern}, + "writes": { + "type": "array", + "items": {"properties": {"doc_id": {"type": "string", "pattern": pattern}}}, + }, + "query": {"anyOf": [{"type": "string", "pattern": pattern}, {"type": "null"}]}, + "pair": {"type": "array", "prefixItems": [{"type": "string", "pattern": pattern}]}, + "extra": {"type": "object", "additionalProperties": {"type": "string", "pattern": pattern}}, + "tagged": { + "type": "object", + "patternProperties": {pattern: {"type": "string"}, "^x_": {"type": "integer"}}, + }, + }, + "$defs": {"segment": {"type": "string", "pattern": pattern}}, + "required": ["field"], + } + + def test_drops_every_regex_python_re_rejects_from_every_schema_position(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + drop_non_python_regex_patterns, + ) + + schema = self._schema(_ARTIFACT_FIELD_PATTERN) + + result = drop_non_python_regex_patterns(schema) + + assert '"pattern"' not in json.dumps(result) + properties = result["properties"] + assert properties["field"] == {"type": "string"} + assert properties["writes"]["items"]["properties"]["doc_id"] == {"type": "string"} + assert properties["query"]["anyOf"] == [{"type": "string"}, {"type": "null"}] + assert properties["pair"]["prefixItems"] == [{"type": "string"}] + assert properties["extra"]["additionalProperties"] == {"type": "string"} + assert properties["tagged"]["patternProperties"] == {"^x_": {"type": "integer"}} + assert result["$defs"]["segment"] == {"type": "string"} + assert result["required"] == ["field"] + assert schema == self._schema(_ARTIFACT_FIELD_PATTERN) + + def test_keeps_regexes_python_re_compiles_and_returns_the_same_object(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + drop_non_python_regex_patterns, + ) + + schema = self._schema(r'^(?!__.*__$)[^"\\./[\]]{1,200}$') + + assert drop_non_python_regex_patterns(schema) is schema + + def test_pattern_keys_inside_data_positions_are_not_regexes(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + drop_non_python_regex_patterns, + ) + + schema = { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "template": {"type": "object", "default": {"pattern": _ARTIFACT_FIELD_PATTERN}}, + "samples": {"type": "array", "examples": [{"pattern": _ARTIFACT_FIELD_PATTERN}]}, + "fixed": {"const": {"pattern": _ARTIFACT_FIELD_PATTERN}}, + "vendor": {"type": "string", "x-litellm": {"pattern": _ARTIFACT_FIELD_PATTERN}}, + }, + "required": ["pattern"], + } + + assert drop_non_python_regex_patterns(schema) is schema + + def test_regex_nested_past_what_python_re_can_parse_is_dropped_not_raised(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + drop_non_python_regex_patterns, + ) + + schema = { + "type": "object", + "properties": {"deep": {"type": "string", "pattern": "(" * 2000 + "a" + ")" * 2000}}, + } + + assert drop_non_python_regex_patterns(schema)["properties"]["deep"] == {"type": "string"} + + def test_walks_schemas_deeper_than_the_interpreter_recursion_limit(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + drop_non_python_regex_patterns, + ) + + depth = sys.getrecursionlimit() + leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} + schema = functools.reduce( + lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(depth), leaf + ) + + result = drop_non_python_regex_patterns(schema) + + assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), result) == {"type": "string"} + assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), schema) is leaf + + def test_leaves_levels_past_the_json_nesting_limit_alone(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + drop_non_python_regex_patterns, + ) + + leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} + schema = functools.reduce( + lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(1100), leaf + ) + + assert drop_non_python_regex_patterns(schema) is schema class TestRequestContainsImageContent: diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py index 4e6b9ed0188..bc6cb0c0fed 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -198,6 +198,9 @@ def test_azure_gpt_5_takes_the_reasoning_path() -> None: assert "reasoning_effort" in supported +_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' + + class TestAzureToolSchemaCombinatorFlattening: """ Regression tests for LIT-6510: Azure's chat completions validator rejects @@ -259,6 +262,26 @@ class TestAzureToolSchemaCombinatorFlattening: self._transform(AzureOpenAIConfig(), "gpt-4o", [tool]) assert tool == self._anyof_tool() + def test_transform_request_drops_non_python_regex_pattern(self): + tool = { + "type": "function", + "function": { + "name": "Artifact", + "parameters": { + "type": "object", + "properties": {"field": {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN}}, + }, + }, + } + + request = self._transform(AzureOpenAIConfig(), "gpt-4o", [tool]) + + assert request["tools"][0]["function"]["parameters"] == { + "type": "object", + "properties": {"field": {"type": "string"}}, + } + assert tool["function"]["parameters"]["properties"]["field"]["pattern"] == _ARTIFACT_FIELD_PATTERN + def test_clean_object_schema_passes_through_as_same_object(self): tool = { "type": "function", diff --git a/tests/test_litellm/llms/azure/response/test_azure_transformation.py b/tests/test_litellm/llms/azure/response/test_azure_transformation.py index 0cac2705ab0..726c9f65681 100644 --- a/tests/test_litellm/llms/azure/response/test_azure_transformation.py +++ b/tests/test_litellm/llms/azure/response/test_azure_transformation.py @@ -1,4 +1,5 @@ from copy import deepcopy +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -243,6 +244,9 @@ def test_provider_config_manager_o_series_selection(): assert not isinstance(default_config, AzureOpenAIOSeriesResponsesAPIConfig) +_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' + + class TestAzureResponsesAPIConfig: def setup_method(self): self.config = AzureOpenAIResponsesAPIConfig() @@ -599,6 +603,31 @@ class TestAzureResponsesAPIConfig: assert result["tools"][0] is tool assert "anyOf" in result["tools"][0]["parameters"] + def test_azure_drops_non_python_regex_pattern_while_keeping_gpt5_combinators(self): + tool = { + "type": "function", + "name": "Artifact", + "parameters": { + "type": "object", + "anyOf": [{"properties": {"field": {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN}}}], + "properties": {"field": {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN}}, + }, + } + + result = self.config.transform_responses_api_request( + model="my-eastus-deployment", + input="hi", + response_api_optional_request_params={"tools": [tool]}, + litellm_params=GenericLiteLLMParams(model_info={"base_model": "azure/gpt-5.4-mini"}), + headers={}, + ) + + assert result["tools"][0]["parameters"] == { + "type": "object", + "anyOf": [{"properties": {"field": {"type": "string"}}}], + "properties": {"field": {"type": "string"}}, + } + def test_azure_keeps_combinators_for_unrecognized_deployment_without_base_model(self): tool = self._anyof_tool() diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index 9737d63cc26..b110586ae5b 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -4,6 +4,7 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation import pytest +from typing import Final import litellm @@ -1168,6 +1169,9 @@ class TestOpenAIPromptCacheBreakpointChatPath: assert "prompt_cache_options" not in request +_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' + + class TestToolSchemaCombinatorFlatteningForOpenAI: """ Regression tests for LIT-6488: OpenAI's chat completions validator rejects @@ -1281,3 +1285,50 @@ class TestToolSchemaCombinatorFlatteningForOpenAI: parameters = request["tools"][0]["function"]["parameters"] assert "anyOf" not in parameters assert set(parameters["properties"]) == {"id", "enabled", "schedule"} + + @staticmethod + def _artifact_tool(): + return { + "type": "function", + "function": { + "name": "Artifact", + "parameters": { + "type": "object", + "properties": {"field": {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN}}, + "required": ["field"], + }, + }, + } + + def test_drops_non_python_regex_pattern_for_hosted_openai(self): + tool = self._artifact_tool() + + request = self._transform(self.config, "gpt-4o", {"custom_llm_provider": "openai", "api_base": None}, [tool]) + + assert request["tools"][0]["function"]["parameters"] == { + "type": "object", + "properties": {"field": {"type": "string"}}, + "required": ["field"], + } + assert tool == self._artifact_tool() + + def test_custom_api_base_drops_non_python_regex_pattern_but_keeps_union(self): + tool = self._anyof_tool() + tool["function"]["parameters"]["properties"]["id"]["pattern"] = _ARTIFACT_FIELD_PATTERN + + request = self._transform( + self.config, "gpt-4o", {"custom_llm_provider": "openai", "api_base": "http://localhost:8000/v1"}, [tool] + ) + + parameters = request["tools"][0]["function"]["parameters"] + assert parameters["properties"]["id"] == {"type": "string"} + assert parameters["anyOf"] == self._anyof_tool()["function"]["parameters"]["anyOf"] + + def test_non_openai_provider_keeps_non_python_regex_pattern(self): + tool = self._artifact_tool() + + request = self._transform( + self.config, "some-oss-model", {"custom_llm_provider": "groq", "api_base": None}, [tool] + ) + + assert request["tools"][0] is tool diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index c5902b32a06..4cf8767764b 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -1,5 +1,6 @@ import json from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx @@ -20,6 +21,8 @@ from litellm.types.llms.openai import ( ) from litellm.types.router import GenericLiteLLMParams +_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' + class TestOpenAIResponsesAPIConfig: def setup_method(self): @@ -2022,6 +2025,86 @@ class TestFlattenToolSchemaCombinatorsWiring: assert "anyOf" not in result["tools"][1]["parameters"] +class TestToolSchemaRegexPatternWiring: + """Claude Code's Artifact tool reaches /v1/responses (the /v1/messages bridge) with an + ECMA-262 ``pattern``; OpenAI compiles patterns with Python ``re`` and 400s + "'...' is not a 'regex'" for every model family, so the keyword is dropped. + """ + + def _artifact_tool(self): + return { + "type": "function", + "name": "Artifact", + "parameters": { + "type": "object", + "properties": { + "field": {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN}, + "doc_id": {"type": "string", "pattern": r"^(?!\.\.?(?:/|$))[A-Za-z0-9_\-.~:@+]{1,200}$"}, + }, + "required": ["field"], + }, + } + + @pytest.mark.parametrize("model", ["gpt-5.6", "gpt-4o", "o3"]) + def test_openai_drops_only_the_pattern_python_re_rejects_for_every_family(self, model): + tool = self._artifact_tool() + + result = OpenAIResponsesAPIConfig().transform_responses_api_request( + model=model, + input="hi", + response_api_optional_request_params={"tools": [tool]}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + properties = result["tools"][0]["parameters"]["properties"] + assert properties["field"] == {"type": "string"} + assert properties["doc_id"] == tool["parameters"]["properties"]["doc_id"] + assert result["tools"][0]["parameters"]["required"] == ["field"] + assert tool["parameters"]["properties"]["field"]["pattern"] == _ARTIFACT_FIELD_PATTERN + assert json.loads(json.dumps(result["tools"])) == result["tools"] + + def test_openai_drops_patterns_inside_codex_namespace_tools(self): + namespace = {"type": "namespace", "name": "mcp__claude", "tools": [self._artifact_tool()]} + + result = OpenAIResponsesAPIConfig().transform_responses_api_request( + model="gpt-5.6", + input="hi", + response_api_optional_request_params={"tools": [namespace]}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["tools"][0]["tools"][0]["parameters"]["properties"]["field"] == {"type": "string"} + + def test_openai_compact_request_drops_patterns(self): + _, data = OpenAIResponsesAPIConfig().transform_compact_response_api_request( + model="gpt-5.6", + input="hi", + response_api_optional_request_params={"tools": [self._artifact_tool()]}, + api_base="https://api.openai.com/v1/responses", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert data["tools"][0]["parameters"]["properties"]["field"] == {"type": "string"} + + def test_non_openai_subclass_keeps_patterns(self): + from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig + + tool = self._artifact_tool() + + result = HostedVLLMResponsesAPIConfig().transform_responses_api_request( + model="hosted_vllm/qwen", + input="hi", + response_api_optional_request_params={"tools": [tool]}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["tools"][0] is tool + + class TestReasoningFollowsModelSupport: """Responses API clients like Codex send `reasoning` on every request, and OpenAI 400s it on non-reasoning models like gpt-4o. drop_params must strip it there, the same way the From dde19adde171c3262388e589b6a3f8c4b352433c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 05:44:30 +0000 Subject: [PATCH 298/411] fix(proxy): bound concurrent key and spend-counter DB lookups to stop prisma pool thrash (#40387) * fix(proxy): bound concurrent key and spend-counter DB lookups to stop prisma pool thrash A cache-miss burst fanned every get_key_object DB fallback and SpendCounterReseed point lookup into the prisma query-engine httpx pool at once. httpcore's request assignment is O(queued x connections) per event, so the event loop spent most of its time in pool bookkeeping and the logging worker's 20s wait_for tripped. Callers now wait on a small shared semaphore (PROXY_DB_LOOKUP_MAX_CONCURRENCY, default 25) instead of queueing inside httpcore Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): type the in-flight counting prisma fake Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): drop module docstring from db_lookup_gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): type the in-flight counting table fake Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/proxy/auth/auth_checks.py | 62 ++++++++++--------- litellm/proxy/db/db_lookup_gate.py | 23 +++++++ litellm/proxy/db/spend_counter_reseed.py | 50 ++++++++------- .../proxy/auth/test_auth_checks.py | 41 +++++++++++- .../proxy/db/test_spend_counter_reseed.py | 31 ++++++++++ 6 files changed, 154 insertions(+), 54 deletions(-) create mode 100644 litellm/proxy/db/db_lookup_gate.py diff --git a/litellm/constants.py b/litellm/constants.py index d545890dd59..0a7ef363e92 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1682,6 +1682,7 @@ SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INT RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, int(os.getenv("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", "3"))) RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = float(os.getenv("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", "0.2")) SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000)) +PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, int(os.getenv("PROXY_DB_LOOKUP_MAX_CONCURRENCY", "25"))) DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)) # 1 minute PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)) RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_BATCH_SIZE", "500"))) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index a71a1993064..1efc9611fe6 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -94,6 +94,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( team_membership_auth_cache_key, team_membership_reservation_cache_key, ) +from litellm.proxy.db.db_lookup_gate import db_lookup_gate from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.guardrails.tool_name_extraction import ( TOOL_CAPABLE_CALL_TYPES, @@ -3450,36 +3451,37 @@ async def _fetch_key_object_from_db_with_reconnect( """ Fetch key object from DB and retry once if a DB connection error can be healed. """ - try: - return await prisma_client.get_data( - token=hashed_token, - table_name="combined_view", - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) - except Exception as e: - if PrismaDBExceptionHandler.is_database_transport_error(e): - did_reconnect = False - if hasattr(prisma_client, "attempt_db_reconnect"): - auth_reconnect_timeout = getattr(prisma_client, "_db_auth_reconnect_timeout_seconds", 2.0) - if not isinstance(auth_reconnect_timeout, (int, float)): - auth_reconnect_timeout = 2.0 - auth_reconnect_lock_timeout = getattr(prisma_client, "_db_auth_reconnect_lock_timeout_seconds", 0.1) - if not isinstance(auth_reconnect_lock_timeout, (int, float)): - auth_reconnect_lock_timeout = 0.1 - did_reconnect = await prisma_client.attempt_db_reconnect( - reason="auth_get_key_object_lookup_failure", - timeout_seconds=auth_reconnect_timeout, - lock_timeout_seconds=auth_reconnect_lock_timeout, - ) - if did_reconnect: - return await prisma_client.get_data( - token=hashed_token, - table_name="combined_view", - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) - raise + async with db_lookup_gate.current(): + try: + return await prisma_client.get_data( + token=hashed_token, + table_name="combined_view", + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception as e: + if PrismaDBExceptionHandler.is_database_transport_error(e): + did_reconnect = False + if hasattr(prisma_client, "attempt_db_reconnect"): + auth_reconnect_timeout = getattr(prisma_client, "_db_auth_reconnect_timeout_seconds", 2.0) + if not isinstance(auth_reconnect_timeout, (int, float)): + auth_reconnect_timeout = 2.0 + auth_reconnect_lock_timeout = getattr(prisma_client, "_db_auth_reconnect_lock_timeout_seconds", 0.1) + if not isinstance(auth_reconnect_lock_timeout, (int, float)): + auth_reconnect_lock_timeout = 0.1 + did_reconnect = await prisma_client.attempt_db_reconnect( + reason="auth_get_key_object_lookup_failure", + timeout_seconds=auth_reconnect_timeout, + lock_timeout_seconds=auth_reconnect_lock_timeout, + ) + if did_reconnect: + return await prisma_client.get_data( + token=hashed_token, + table_name="combined_view", + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + raise def jwt_key_mapping_cache_key(jwt_claim_name: str, jwt_claim_value: str) -> str: diff --git a/litellm/proxy/db/db_lookup_gate.py b/litellm/proxy/db/db_lookup_gate.py new file mode 100644 index 00000000000..2fd427687bd --- /dev/null +++ b/litellm/proxy/db/db_lookup_gate.py @@ -0,0 +1,23 @@ +import asyncio +from typing import Final + +from litellm.constants import PROXY_DB_LOOKUP_MAX_CONCURRENCY + + +class LoopBoundSemaphore: + __slots__ = ("_loop", "_semaphore", "_value") + + def __init__(self, value: int) -> None: + self._value: Final = value + self._loop: asyncio.AbstractEventLoop | None = None + self._semaphore: asyncio.Semaphore | None = None + + def current(self) -> asyncio.Semaphore: + loop: Final = asyncio.get_running_loop() + if self._semaphore is None or self._loop is not loop: + self._semaphore = asyncio.Semaphore(self._value) + self._loop = loop + return self._semaphore + + +db_lookup_gate: Final = LoopBoundSemaphore(PROXY_DB_LOOKUP_MAX_CONCURRENCY) diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index a38b8a47dbd..e35b1c8c82b 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -23,6 +23,7 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.proxy._types import Litellm_EntityType +from litellm.proxy.db.db_lookup_gate import db_lookup_gate from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.table_repositories import ( BudgetWindowSpendRepository, @@ -121,30 +122,33 @@ class SpendCounterReseed: if SpendCounterReseed._is_key_or_team_window_counter(counter_key): return None try: - if counter_key.startswith("spend:key:"): - token: Final = counter_key[len("spend:key:") :] - row = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": token}) - elif counter_key.startswith("spend:team_member:"): - suffix: Final = counter_key[len("spend:team_member:") :] - if ":" not in suffix: + async with db_lookup_gate.current(): + if counter_key.startswith("spend:key:"): + token: Final = counter_key[len("spend:key:") :] + row = await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": token}) + elif counter_key.startswith("spend:team_member:"): + suffix: Final = counter_key[len("spend:team_member:") :] + if ":" not in suffix: + return None + user_id, team_id = suffix.rsplit(":", 1) + row = await TeamMembershipRepository(prisma_client).table.find_unique( + where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}} + ) + elif counter_key.startswith("spend:team:"): + team_id = counter_key[len("spend:team:") :] + row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) + elif counter_key.startswith("spend:user:"): + user_id = counter_key[len("spend:user:") :] + row = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_id}) + elif counter_key.startswith(END_USER_COUNTER_PREFIX) or counter_key.startswith("spend:tag:"): + return None + elif counter_key.startswith("spend:org:"): + org_id: Final = counter_key[len("spend:org:") :] + row = await OrganizationRepository(prisma_client).table.find_unique( + where={"organization_id": org_id} + ) + else: return None - user_id, team_id = suffix.rsplit(":", 1) - row = await TeamMembershipRepository(prisma_client).table.find_unique( - where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}} - ) - elif counter_key.startswith("spend:team:"): - team_id = counter_key[len("spend:team:") :] - row = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id}) - elif counter_key.startswith("spend:user:"): - user_id = counter_key[len("spend:user:") :] - row = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_id}) - elif counter_key.startswith(END_USER_COUNTER_PREFIX) or counter_key.startswith("spend:tag:"): - return None - elif counter_key.startswith("spend:org:"): - org_id: Final = counter_key[len("spend:org:") :] - row = await OrganizationRepository(prisma_client).table.find_unique(where={"organization_id": org_id}) - else: - return None except Exception: verbose_proxy_logger.exception("SpendCounterReseed.from_db: failed for %s", counter_key) return None diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 5bfef2b6445..8777e24e209 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -1,7 +1,7 @@ import asyncio import json from types import SimpleNamespace -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch if TYPE_CHECKING: @@ -37,6 +37,7 @@ from litellm.proxy.auth.auth_checks import ( _can_object_call_vector_stores, _check_end_user_budget, _check_team_member_budget, + _fetch_key_object_from_db_with_reconnect, _get_fuzzy_user_object, _get_team_db_check, _log_budget_lookup_failure, @@ -55,6 +56,7 @@ from litellm.caching.redis_cache import RedisCache from litellm.constants import ( DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, END_USER_RESTRICTED_REGISTRY_MAX_SIZE, + PROXY_DB_LOOKUP_MAX_CONCURRENCY, REGISTRY_ERROR_NEGATIVE_CACHE_TTL, TAG_REGISTRY_MAX_SIZE, ) @@ -564,6 +566,43 @@ async def test_get_key_object_should_raise_if_reconnect_fails_on_db_connection_e assert mock_prisma_client.get_data.await_count == 1 +class _InFlightCountingPrisma: + def __init__(self) -> None: + self.in_flight = 0 + self.max_in_flight = 0 + + async def get_data( + self, token: str, table_name: str, parent_otel_span: None, proxy_logging_obj: None + ) -> UserAPIKeyAuth: + self.in_flight += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + await asyncio.sleep(0.001) + self.in_flight -= 1 + return UserAPIKeyAuth(token=token) + + +@pytest.mark.asyncio +async def test_fetch_key_object_from_db_bounds_in_flight_prisma_requests(): + prisma: Final = _InFlightCountingPrisma() + burst: Final = PROXY_DB_LOOKUP_MAX_CONCURRENCY * 5 + + results: Final = await asyncio.gather( + *( + _fetch_key_object_from_db_with_reconnect( + hashed_token=f"hashed-token-{i}", + prisma_client=prisma, # pyright: ignore[reportArgumentType] # fake stands in for PrismaClient + parent_otel_span=None, + proxy_logging_obj=None, + ) + for i in range(burst) + ) + ) + + assert len(results) == burst + assert {r.token for r in results if r is not None} == {f"hashed-token-{i}" for i in range(burst)} + assert prisma.max_in_flight == PROXY_DB_LOOKUP_MAX_CONCURRENCY + + def _fake_redis_cache(): fake_redis = MagicMock() fake_redis.async_get_cache = AsyncMock(return_value=None) diff --git a/tests/test_litellm/proxy/db/test_spend_counter_reseed.py b/tests/test_litellm/proxy/db/test_spend_counter_reseed.py index 8cb3fc665eb..0361d5cfe8f 100644 --- a/tests/test_litellm/proxy/db/test_spend_counter_reseed.py +++ b/tests/test_litellm/proxy/db/test_spend_counter_reseed.py @@ -7,6 +7,7 @@ allowed to run: only when the row is missing or belongs to an older window. from __future__ import annotations +import asyncio from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final @@ -14,6 +15,7 @@ from typing import Final import pytest from litellm.caching.dual_cache import DualCache +from litellm.constants import PROXY_DB_LOOKUP_MAX_CONCURRENCY from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed WINDOW_START = datetime(2026, 8, 1, tzinfo=timezone.utc) @@ -42,6 +44,19 @@ class _FakeSpendLogsTable: return [{by[0]: where.get(by[0]), "_sum": {"spend": self._total}}] +class _InFlightCountingTable: + def __init__(self) -> None: + self.in_flight = 0 + self.max_in_flight = 0 + + async def find_unique(self, where: dict[str, str]) -> SimpleNamespace: + self.in_flight += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + await asyncio.sleep(0.001) + self.in_flight -= 1 + return SimpleNamespace(token=where["token"], spend=1.0) + + class _FakePrismaClient: def __init__( self, @@ -55,6 +70,7 @@ class _FakePrismaClient: litellm_budgetwindowspend=_FakeFindUniqueTable(row=row, error=error), litellm_spendlogs=_FakeSpendLogsTable(total=spend_logs_total), litellm_endusertable=_FakeFindUniqueTable(row=end_user_row, error=end_user_error), + litellm_verificationtoken=_InFlightCountingTable(), ) @@ -306,6 +322,21 @@ async def test_end_user_from_db_returns_none_without_a_row_a_client_or_on_db_err ) +@pytest.mark.asyncio +async def test_from_db_bounds_in_flight_prisma_requests_across_counter_keys(): + """Per-counter singleflight only collapses duplicates of one key. A cold-cache burst + over many distinct keys must still not flood the prisma engine HTTP pool (LIT-6435).""" + prisma: Final = _FakePrismaClient() + burst: Final = PROXY_DB_LOOKUP_MAX_CONCURRENCY * 5 + + results: Final = await asyncio.gather( + *(SpendCounterReseed.from_db(prisma_client=prisma, counter_key=f"spend:key:hashed-{i}") for i in range(burst)) + ) + + assert results == [1.0] * burst + assert prisma.db.litellm_verificationtoken.max_in_flight == PROXY_DB_LOOKUP_MAX_CONCURRENCY + + @pytest.mark.asyncio async def test_from_db_still_never_reads_the_end_user_row(): """A cold end-user counter keeps seeding from the cached end-user object the auth From 6bb60f34e342ebac3bc01cab70820cf83acf9dc2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:14:16 -0700 Subject: [PATCH 299/411] fix(s3_v2): freeze refreshable credentials before signing and retry 403 uploads with a fresh signature (#40187) RefreshableCredentials (IMDS roles) can refresh between the access key, secret and token reads SigV4 performs, producing a mixed-generation signature that S3 rejects with 403 and the log is dropped. Snapshot the credentials with get_frozen_credentials before signing, treat 403 like 500/503 in the upload retry loop, and fetch credentials plus sign again on every attempt in both the async and sync upload paths. Tests load a real botocore credential_process profile and fake only the HTTP boundary with httpx.MockTransport Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/s3_v2.py | 104 ++-- tests/test_litellm/integrations/test_s3_v2.py | 443 ++++++++++++------ 2 files changed, 348 insertions(+), 199 deletions(-) diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 996ff9c75af..972ac79e306 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -10,9 +10,11 @@ import asyncio import time from collections.abc import Mapping from datetime import datetime -from typing import Final, cast +from typing import TYPE_CHECKING, Final, cast from urllib.parse import quote +import httpx + import litellm from litellm._logging import print_verbose, verbose_logger from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS @@ -35,6 +37,9 @@ from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload from .custom_batch_logger import CustomBatchLogger +if TYPE_CHECKING: + from botocore.credentials import Credentials + class S3Logger(CustomBatchLogger, BaseAWSLLM): def __init__( @@ -232,6 +237,26 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): f"{get_aws_dns_suffix(self.s3_region_name)}/{encoded_key}" ) + def _sign_put( + self, credentials: "Credentials", url: str, json_string: str, headers: Mapping[str, str] + ) -> dict[str, str]: # mutable-ok: [LIT001] AsyncHTTPHandler.put/HTTPHandler.put only accept dict headers + """ + ``RefreshableCredentials`` (IMDS roles) may refresh between the access key, secret and token + reads SigV4 performs, producing a mixed-generation signature that S3 rejects with 403. + Freezing first makes the three values one atomic snapshot. + """ + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import RefreshableCredentials + + frozen: Final = ( + credentials.get_frozen_credentials() if isinstance(credentials, RefreshableCredentials) else credentials + ) + aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=dict(headers)) + aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name) + S3SigV4Auth(frozen, "s3", aws_region_name).add_auth(aws_request) + return dict(aws_request.headers.items()) + def _sse_headers(self) -> Mapping[str, str]: candidates: Final = { "x-amz-server-side-encryption": self.s3_server_side_encryption, @@ -317,26 +342,12 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): try: import base64 import hashlib - - from botocore.auth import S3SigV4Auth - from botocore.awsrequest import AWSRequest except ImportError: raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") try: from litellm.litellm_core_utils.asyncify import asyncify asyncified_get_credentials: Final = asyncify(self.get_credentials) - credentials: Final = await asyncified_get_credentials( - aws_access_key_id=self.s3_aws_access_key_id, - aws_secret_access_key=self.s3_aws_secret_access_key, - aws_session_token=self.s3_aws_session_token, - aws_region_name=self.s3_region_name, - aws_session_name=self.s3_aws_session_name, - aws_profile_name=self.s3_aws_profile_name, - aws_role_name=self.s3_aws_role_name, - aws_web_identity_token=self.s3_aws_web_identity_token, - aws_sts_endpoint=self.s3_aws_sts_endpoint, - ) verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key) verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify) @@ -363,19 +374,28 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): **self._sse_headers(), } - # Sign the request - aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers) - aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name) - await run_aws_signing(S3SigV4Auth(credentials, "s3", aws_region_name).add_auth, aws_request) + async def signed_put() -> httpx.Response: + credentials: Final = await asyncified_get_credentials( + aws_access_key_id=self.s3_aws_access_key_id, + aws_secret_access_key=self.s3_aws_secret_access_key, + aws_session_token=self.s3_aws_session_token, + aws_region_name=self.s3_region_name, + aws_session_name=self.s3_aws_session_name, + aws_profile_name=self.s3_aws_profile_name, + aws_role_name=self.s3_aws_role_name, + aws_web_identity_token=self.s3_aws_web_identity_token, + aws_sts_endpoint=self.s3_aws_sts_endpoint, + ) + signed_headers: Final = await run_aws_signing(self._sign_put, credentials, url, json_string, headers) + try: + return await self.async_httpx_client.put(url, data=json_string, headers=signed_headers) + except httpx.HTTPStatusError as error: + return error.response - # Prepare the signed headers - signed_headers: Final = dict(aws_request.headers.items()) - - # Make the request with retry for transient S3 errors (500/503) max_retries: Final = 3 for attempt in range(max_retries): - response = await self.async_httpx_client.put(url, data=json_string, headers=signed_headers) - if response.status_code in (500, 503) and attempt < max_retries - 1: + response = await signed_put() + if response.status_code in (403, 500, 503) and attempt < max_retries - 1: wait_time = 2**attempt # 1s, 2s verbose_logger.warning( "S3 upload returned %s, retrying in %ss (attempt %s/%s) key=%s", @@ -479,20 +499,10 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): try: import base64 import hashlib - - from botocore.auth import S3SigV4Auth - from botocore.awsrequest import AWSRequest - from botocore.credentials import Credentials except ImportError: raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") try: verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key) - credentials: Final[Credentials] = self.get_credentials( - aws_access_key_id=self.s3_aws_access_key_id, - aws_secret_access_key=self.s3_aws_secret_access_key, - aws_session_token=self.s3_aws_session_token, - aws_region_name=self.s3_region_name, - ) url: Final = self._build_object_url(batch_logging_element.s3_object_key) @@ -516,22 +526,24 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): **self._sse_headers(), } - # Sign the request - aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers) - aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name) - S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request) - - # Prepare the signed headers - signed_headers: Final = dict(aws_request.headers.items()) - httpx_client: Final = _get_httpx_client( params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None) ) - # Make the request with retry for transient S3 errors (500/503) + + def signed_put() -> httpx.Response: + credentials: Final = self.get_credentials( + aws_access_key_id=self.s3_aws_access_key_id, + aws_secret_access_key=self.s3_aws_secret_access_key, + aws_session_token=self.s3_aws_session_token, + aws_region_name=self.s3_region_name, + ) + signed_headers: Final = self._sign_put(credentials, url, json_string, headers) + return httpx_client.put(url, data=json_string, headers=signed_headers) + max_retries: Final = 3 for attempt in range(max_retries): - response = httpx_client.put(url, data=json_string, headers=signed_headers) - if response.status_code in (500, 503) and attempt < max_retries - 1: + response = signed_put() + if response.status_code in (403, 500, 503) and attempt < max_retries - 1: wait_time = 2**attempt # 1s, 2s verbose_logger.warning( "S3 upload returned %s, retrying in %ss (attempt %s/%s) key=%s", diff --git a/tests/test_litellm/integrations/test_s3_v2.py b/tests/test_litellm/integrations/test_s3_v2.py index a037284d7c1..08d37297ab1 100644 --- a/tests/test_litellm/integrations/test_s3_v2.py +++ b/tests/test_litellm/integrations/test_s3_v2.py @@ -1,10 +1,19 @@ import asyncio +import re +import sys +import textwrap +import uuid +from contextlib import asynccontextmanager from datetime import datetime -from unittest.mock import MagicMock, patch +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, call, patch +import httpx import pytest from litellm.integrations.s3_v2 import S3Logger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.integrations.s3_v2 import s3BatchLoggingElement from litellm.types.utils import StandardLoggingPayload @@ -21,9 +30,7 @@ class TestS3V2UnitTests: source_code = inspect.getsource(s3_v2) # Verify that json.dumps is not used directly in the code - assert ( - "json.dumps(" not in source_code - ), "S3 v2 should not use json.dumps directly" + assert "json.dumps(" not in source_code, "S3 v2 should not use json.dumps directly" @patch("asyncio.create_task") @patch("litellm.integrations.s3_v2.CustomBatchLogger.periodic_flush") @@ -86,12 +93,8 @@ class TestS3V2UnitTests: call_args_minio = s3_logger_minio.async_httpx_client.put.call_args assert call_args_minio is not None url_minio = call_args_minio[0][0] - expected_minio_url = ( - "https://minio.example.com:9000/litellm-logs/2025-09-14/test-key.json" - ) - assert ( - url_minio == expected_minio_url - ), f"Expected MinIO URL {expected_minio_url}, got {url_minio}" + expected_minio_url = "https://minio.example.com:9000/litellm-logs/2025-09-14/test-key.json" + assert url_minio == expected_minio_url, f"Expected MinIO URL {expected_minio_url}, got {url_minio}" # Test 3: Custom endpoint without bucket name (should fall back to default) s3_logger_no_bucket = S3Logger( @@ -136,12 +139,8 @@ class TestS3V2UnitTests: call_args_sync = mock_sync_client.put.call_args assert call_args_sync is not None url_sync = call_args_sync[0][0] - expected_sync_url = ( - "https://custom.s3.endpoint.com/sync-bucket/2025-09-14/test-key.json" - ) - assert ( - url_sync == expected_sync_url - ), f"Expected sync URL {expected_sync_url}, got {url_sync}" + expected_sync_url = "https://custom.s3.endpoint.com/sync-bucket/2025-09-14/test-key.json" + assert url_sync == expected_sync_url, f"Expected sync URL {expected_sync_url}, got {url_sync}" # Test 5: Download method with custom endpoint s3_logger_download = S3Logger( @@ -158,19 +157,15 @@ class TestS3V2UnitTests: s3_logger_download.async_httpx_client = AsyncMock() s3_logger_download.async_httpx_client.get.return_value = mock_download_response - result = asyncio.run( - s3_logger_download._download_object_from_s3( - "2025-09-14/download-test-key.json" - ) - ) + result = asyncio.run(s3_logger_download._download_object_from_s3("2025-09-14/download-test-key.json")) call_args_download = s3_logger_download.async_httpx_client.get.call_args assert call_args_download is not None url_download = call_args_download[0][0] expected_download_url = "https://download.s3.endpoint.com/download-bucket/2025-09-14/download-test-key.json" - assert ( - url_download == expected_download_url - ), f"Expected download URL {expected_download_url}, got {url_download}" + assert url_download == expected_download_url, ( + f"Expected download URL {expected_download_url}, got {url_download}" + ) assert result == {"downloaded": "data"} @@ -216,12 +211,8 @@ class TestS3V2UnitTests: call_args = s3_logger_virtual.async_httpx_client.put.call_args assert call_args is not None url = call_args[0][0] - expected_url = ( - "https://test-bucket.s3.custom-endpoint.com/2025-09-14/test-key.json" - ) - assert ( - url == expected_url - ), f"Expected virtual-hosted-style URL {expected_url}, got {url}" + expected_url = "https://test-bucket.s3.custom-endpoint.com/2025-09-14/test-key.json" + assert url == expected_url, f"Expected virtual-hosted-style URL {expected_url}, got {url}" # Test 2: Path-style (default behavior with s3_use_virtual_hosted_style=False) s3_logger_path = S3Logger( @@ -241,12 +232,8 @@ class TestS3V2UnitTests: call_args_path = s3_logger_path.async_httpx_client.put.call_args assert call_args_path is not None url_path = call_args_path[0][0] - expected_path_url = ( - "https://s3.custom-endpoint.com/test-bucket/2025-09-14/test-key.json" - ) - assert ( - url_path == expected_path_url - ), f"Expected path-style URL {expected_path_url}, got {url_path}" + expected_path_url = "https://s3.custom-endpoint.com/test-bucket/2025-09-14/test-key.json" + assert url_path == expected_path_url, f"Expected path-style URL {expected_path_url}, got {url_path}" # Test 3: Virtual-hosted-style with http protocol s3_logger_http = S3Logger( @@ -266,12 +253,10 @@ class TestS3V2UnitTests: call_args_http = s3_logger_http.async_httpx_client.put.call_args assert call_args_http is not None url_http = call_args_http[0][0] - expected_http_url = ( - "http://http-bucket.minio.local:9000/2025-09-14/test-key.json" + expected_http_url = "http://http-bucket.minio.local:9000/2025-09-14/test-key.json" + assert url_http == expected_http_url, ( + f"Expected virtual-hosted-style URL with http {expected_http_url}, got {url_http}" ) - assert ( - url_http == expected_http_url - ), f"Expected virtual-hosted-style URL with http {expected_http_url}, got {url_http}" # Test 4: Sync upload method with virtual-hosted-style s3_logger_sync_virtual = S3Logger( @@ -295,12 +280,10 @@ class TestS3V2UnitTests: call_args_sync = mock_sync_client.put.call_args assert call_args_sync is not None url_sync = call_args_sync[0][0] - expected_sync_url = ( - "https://sync-bucket.storage.example.com/2025-09-14/test-key.json" + expected_sync_url = "https://sync-bucket.storage.example.com/2025-09-14/test-key.json" + assert url_sync == expected_sync_url, ( + f"Expected virtual-hosted-style sync URL {expected_sync_url}, got {url_sync}" ) - assert ( - url_sync == expected_sync_url - ), f"Expected virtual-hosted-style sync URL {expected_sync_url}, got {url_sync}" # Test 5: Download method with virtual-hosted-style s3_logger_download_virtual = S3Logger( @@ -316,34 +299,27 @@ class TestS3V2UnitTests: mock_download_response.status_code = 200 mock_download_response.json = MagicMock(return_value={"downloaded": "data"}) s3_logger_download_virtual.async_httpx_client = AsyncMock() - s3_logger_download_virtual.async_httpx_client.get.return_value = ( - mock_download_response - ) + s3_logger_download_virtual.async_httpx_client.get.return_value = mock_download_response - result = asyncio.run( - s3_logger_download_virtual._download_object_from_s3( - "2025-09-14/download-test-key.json" - ) - ) + result = asyncio.run(s3_logger_download_virtual._download_object_from_s3("2025-09-14/download-test-key.json")) call_args_download = s3_logger_download_virtual.async_httpx_client.get.call_args assert call_args_download is not None url_download = call_args_download[0][0] expected_download_url = "https://download-bucket.download.endpoint.com/2025-09-14/download-test-key.json" - assert ( - url_download == expected_download_url - ), f"Expected virtual-hosted-style download URL {expected_download_url}, got {url_download}" + assert url_download == expected_download_url, ( + f"Expected virtual-hosted-style download URL {expected_download_url}, got {url_download}" + ) assert result == {"downloaded": "data"} @patch("asyncio.create_task") @patch("litellm.integrations.s3_v2.CustomBatchLogger.periodic_flush") - def test_s3_v2_put_url_encodes_spaces_in_object_key( - self, mock_periodic_flush, mock_create_task - ): - import requests + def test_s3_v2_put_url_encodes_spaces_in_object_key(self, mock_periodic_flush, mock_create_task): from unittest.mock import AsyncMock + import requests + from litellm.types.integrations.s3_v2 import s3BatchLoggingElement mock_periodic_flush.return_value = None @@ -487,9 +463,7 @@ async def test_async_upload_exhausts_retries_on_persistent_503(): # All 3 attempts return 503 response_503 = MagicMock() response_503.status_code = 503 - response_503.raise_for_status = MagicMock( - side_effect=Exception("503 Service Unavailable") - ) + response_503.raise_for_status = MagicMock(side_effect=Exception("503 Service Unavailable")) logger.async_httpx_client = AsyncMock() logger.async_httpx_client.put = AsyncMock(return_value=response_503) @@ -528,12 +502,12 @@ async def test_async_upload_no_retry_on_4xx(): s3_object_download_filename="test-no-retry.json", ) - response_403 = MagicMock() - response_403.status_code = 403 - response_403.raise_for_status = MagicMock(side_effect=Exception("403 Forbidden")) + response_400 = MagicMock() + response_400.status_code = 400 + response_400.raise_for_status = MagicMock(side_effect=Exception("400 Bad Request")) logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = AsyncMock(return_value=response_403) + logger.async_httpx_client.put = AsyncMock(return_value=response_400) with patch.object(logger, "handle_callback_failure") as mock_failure: await logger.async_upload_data_to_s3(test_element) @@ -543,6 +517,190 @@ async def test_async_upload_no_retry_on_4xx(): mock_failure.assert_called_once_with(callback_name="S3Logger") +_SIGV4_ACCESS_KEY = re.compile(r"Credential=(AKIA\d+)/") + + +@pytest.fixture +def rotating_profile(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str: + """ + A real botocore profile whose credential_process hands out a new key generation on every call and + expires inside the advisory refresh window, so RefreshableCredentials re-runs it on every property read. + """ + counter = tmp_path / "generation" + script = tmp_path / "rotate_credentials.py" + script.write_text( + textwrap.dedent( + f""" + import json, sys + from datetime import datetime, timedelta, timezone + from pathlib import Path + + counter = Path({str(counter)!r}) + generation = int(counter.read_text()) if counter.exists() else 0 + counter.write_text(str(generation + 1)) + expiry = (datetime.now(timezone.utc) + timedelta(minutes=12)).strftime("%Y-%m-%dT%H:%M:%SZ") + json.dump( + {{ + "Version": 1, + "AccessKeyId": f"AKIA{{generation}}", + "SecretAccessKey": f"secret-{{generation}}", + "SessionToken": f"token-{{generation}}", + "Expiration": expiry, + }}, + sys.stdout, + ) + """ + ) + ) + profile = f"rotating-{uuid.uuid4().hex}" + (tmp_path / "config").write_text(f"[profile {profile}]\ncredential_process = {sys.executable} {script}\n") + monkeypatch.setenv("AWS_CONFIG_FILE", str(tmp_path / "config")) + return profile + + +def _generation(request: httpx.Request) -> tuple[str, str]: + """(access key generation, session token generation) SigV4 baked into one request.""" + access_key = _SIGV4_ACCESS_KEY.search(request.headers["Authorization"]) + assert access_key is not None + return access_key.group(1).removeprefix("AKIA"), request.headers["X-Amz-Security-Token"].removeprefix("token-") + + +@asynccontextmanager +async def _s3_logger_on_production_handler(profile: str, statuses: list[int]): + """ + S3Logger wired to the real AsyncHTTPHandler over an httpx MockTransport that answers with the given + statuses in order, so the handler's own raise_for_status behaviour is exercised end to end. + """ + requests: list[httpx.Request] = [] + replies = iter(statuses) + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(next(replies), request=request, text="SignatureDoesNotMatch") + + handler = AsyncHTTPHandler() + handler.client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) + logger = S3Logger( + s3_bucket_name="test-bucket", + s3_region_name="us-east-1", + s3_aws_profile_name=profile, + s3_flush_interval=3600, + ) + logger.async_httpx_client = handler + with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + yield logger, requests, mock_sleep + await handler.client.aclose() + + +@pytest.mark.asyncio +async def test_async_upload_signs_with_one_frozen_credential_snapshot(rotating_profile: str, caplog): + """ + RefreshableCredentials refreshes on every property read once inside the advisory window, so signing + off the live object would mix the access key of one generation with the token of the next. + """ + test_element = s3BatchLoggingElement( + s3_object_key="2025-09-14/test-frozen.json", + payload={"test": "frozen"}, + s3_object_download_filename="test-frozen.json", + ) + async with _s3_logger_on_production_handler(rotating_profile, [200]) as (logger, requests, _): + await logger.async_upload_data_to_s3(test_element) + + assert len(requests) == 1 + access_key_generation, token_generation = _generation(requests[0]) + assert access_key_generation == token_generation + assert "Error uploading to s3" not in caplog.text + + +@pytest.mark.asyncio +async def test_async_upload_retries_403_with_fresh_credentials_and_signature(rotating_profile: str, caplog): + """ + A 403 (SignatureDoesNotMatch after an IMDS rotation) must be retried, and the retry must fetch + credentials again and carry a signature computed from that newer generation. + """ + test_element = s3BatchLoggingElement( + s3_object_key="2025-09-14/test-403.json", + payload={"test": "403"}, + s3_object_download_filename="test-403.json", + ) + async with _s3_logger_on_production_handler(rotating_profile, [403, 200]) as (logger, requests, mock_sleep): + await logger.async_upload_data_to_s3(test_element) + + assert len(requests) == 2 + first_key, first_token = _generation(requests[0]) + second_key, second_token = _generation(requests[1]) + assert first_key == first_token + assert second_key == second_token + assert int(second_key) > int(first_key) + assert requests[1].headers["Authorization"] != requests[0].headers["Authorization"] + mock_sleep.assert_awaited_once_with(1) + assert "Error uploading to s3" not in caplog.text + + +@pytest.mark.asyncio +async def test_async_upload_exhausts_403_retries_through_production_http_handler(rotating_profile: str, caplog): + test_element = s3BatchLoggingElement( + s3_object_key="2025-09-14/test-403-exhausted.json", + payload={"test": "403-exhausted"}, + s3_object_download_filename="test-403-exhausted.json", + ) + async with _s3_logger_on_production_handler(rotating_profile, [403, 403, 403]) as (logger, requests, mock_sleep): + await logger.async_upload_data_to_s3(test_element) + + assert len(requests) == 3 + assert mock_sleep.await_args_list == [call(1), call(2)] + assert "Error uploading to s3" in caplog.text + + +@pytest.mark.asyncio +async def test_async_upload_does_not_retry_404_through_production_http_handler(rotating_profile: str, caplog): + test_element = s3BatchLoggingElement( + s3_object_key="2025-09-14/test-404.json", + payload={"test": "404"}, + s3_object_download_filename="test-404.json", + ) + async with _s3_logger_on_production_handler(rotating_profile, [404]) as (logger, requests, mock_sleep): + await logger.async_upload_data_to_s3(test_element) + + assert len(requests) == 1 + mock_sleep.assert_not_awaited() + assert "Error uploading to s3" in caplog.text + + +def test_sync_upload_retries_403_with_fresh_signature(rotating_profile: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AWS_PROFILE", rotating_profile) + logger = S3Logger(s3_bucket_name="test-bucket", s3_region_name="us-east-1", s3_flush_interval=3600) + test_element = s3BatchLoggingElement( + s3_object_key="2025-09-14/test-sync-403.json", + payload={"test": "sync-403"}, + s3_object_download_filename="test-sync-403.json", + ) + requests: list[httpx.Request] = [] + replies = iter([403, 200]) + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(next(replies), request=request) + + handler = HTTPHandler() + handler.client = httpx.Client(transport=httpx.MockTransport(respond)) + with ( + patch( # test-quality-ok: sync upload builds its HTTPHandler per call, there is no injection seam for it + "litellm.integrations.s3_v2._get_httpx_client", return_value=handler + ), + patch("time.sleep") as mock_sleep, + ): + logger.upload_data_to_s3(test_element) + + assert len(requests) == 2 + first_key, first_token = _generation(requests[0]) + second_key, second_token = _generation(requests[1]) + assert first_key == first_token + assert second_key == second_token + assert int(second_key) > int(first_key) + mock_sleep.assert_called_once_with(1) + + def test_sync_upload_retries_on_s3_503(): """ Test that the sync upload_data_to_s3 retries on transient S3 503. @@ -626,9 +784,7 @@ async def test_async_log_event_skips_when_standard_logging_object_missing(): # Nothing should have been queued (catches the case where code falls # through without returning and appends None to the queue) - assert ( - len(logger.log_queue) == 0 - ), "log_queue should be empty when standard_logging_object is missing" + assert len(logger.log_queue) == 0, "log_queue should be empty when standard_logging_object is missing" @pytest.mark.asyncio @@ -767,20 +923,18 @@ async def test_s3_verify_false_handling(monkeypatch: pytest.MonkeyPatch): litellm, "s3_callback_params", { - "s3_bucket_name": "test-bucket", - "s3_endpoint_url": "https://localhost:443", - "s3_aws_access_key_id": "minioadmin", - "s3_aws_secret_access_key": "minioadmin", - "s3_region_name": "us-east-1", - "s3_verify": False, # This should NOT be ignored - "s3_use_ssl": False, # This should also NOT be ignored - }, + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": "https://localhost:443", + "s3_aws_access_key_id": "minioadmin", + "s3_aws_secret_access_key": "minioadmin", + "s3_region_name": "us-east-1", + "s3_verify": False, # This should NOT be ignored + "s3_use_ssl": False, # This should also NOT be ignored + }, ) with patch("asyncio.create_task"): - with patch( - "litellm.integrations.s3_v2.get_async_httpx_client" - ) as mock_get_client: + with patch("litellm.integrations.s3_v2.get_async_httpx_client") as mock_get_client: mock_client = AsyncMock() mock_get_client.return_value = mock_client @@ -788,22 +942,16 @@ async def test_s3_verify_false_handling(monkeypatch: pytest.MonkeyPatch): logger = S3Logger() # Verify s3_verify is False, not None - assert ( - logger.s3_verify is False - ), f"Expected s3_verify=False, got {logger.s3_verify}" - assert ( - logger.s3_use_ssl is False - ), f"Expected s3_use_ssl=False, got {logger.s3_use_ssl}" + assert logger.s3_verify is False, f"Expected s3_verify=False, got {logger.s3_verify}" + assert logger.s3_use_ssl is False, f"Expected s3_use_ssl=False, got {logger.s3_use_ssl}" # Verify that get_async_httpx_client was called with ssl_verify=False mock_get_client.assert_called_once() call_kwargs = mock_get_client.call_args.kwargs - assert ( - "params" in call_kwargs - ), "params should be passed to get_async_httpx_client" - assert call_kwargs["params"] == { - "ssl_verify": False - }, f"Expected ssl_verify=False in params, got {call_kwargs.get('params')}" + assert "params" in call_kwargs, "params should be passed to get_async_httpx_client" + assert call_kwargs["params"] == {"ssl_verify": False}, ( + f"Expected ssl_verify=False in params, got {call_kwargs.get('params')}" + ) @pytest.mark.asyncio @@ -820,17 +968,15 @@ async def test_s3_verify_none_handling(monkeypatch: pytest.MonkeyPatch): litellm, "s3_callback_params", { - "s3_bucket_name": "test-bucket", - "s3_aws_access_key_id": "test-key", - "s3_aws_secret_access_key": "test-secret", - "s3_region_name": "us-east-1", - }, + "s3_bucket_name": "test-bucket", + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + "s3_region_name": "us-east-1", + }, ) with patch("asyncio.create_task"): - with patch( - "litellm.integrations.s3_v2.get_async_httpx_client" - ) as mock_get_client: + with patch("litellm.integrations.s3_v2.get_async_httpx_client") as mock_get_client: mock_client = AsyncMock() mock_get_client.return_value = mock_client @@ -838,9 +984,7 @@ async def test_s3_verify_none_handling(monkeypatch: pytest.MonkeyPatch): logger = S3Logger() # Verify s3_verify is None (default) - assert ( - logger.s3_verify is None - ), f"Expected s3_verify=None, got {logger.s3_verify}" + assert logger.s3_verify is None, f"Expected s3_verify=None, got {logger.s3_verify}" # Verify that get_async_httpx_client was called mock_get_client.assert_called_once() @@ -868,13 +1012,13 @@ async def test_s3_verify_false_creates_httpx_client_with_verify_false(monkeypatc litellm, "s3_callback_params", { - "s3_bucket_name": "test-bucket", - "s3_endpoint_url": "https://localhost:443", - "s3_aws_access_key_id": "minioadmin", - "s3_aws_secret_access_key": "minioadmin", - "s3_region_name": "us-east-1", - "s3_verify": False, - }, + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": "https://localhost:443", + "s3_aws_access_key_id": "minioadmin", + "s3_aws_secret_access_key": "minioadmin", + "s3_region_name": "us-east-1", + "s3_verify": False, + }, ) with patch("asyncio.create_task"): @@ -890,9 +1034,7 @@ async def test_s3_verify_false_creates_httpx_client_with_verify_false(monkeypatc httpx_client = logger.async_httpx_client.client # Check the _verify attribute (httpx internal) if hasattr(httpx_client, "_verify"): - assert ( - httpx_client._verify is False - ), f"Expected httpx client _verify=False, got {httpx_client._verify}" + assert httpx_client._verify is False, f"Expected httpx client _verify=False, got {httpx_client._verify}" @pytest.mark.asyncio @@ -910,13 +1052,13 @@ async def test_s3_verify_false_async_client(monkeypatch: pytest.MonkeyPatch): litellm, "s3_callback_params", { - "s3_bucket_name": "test-bucket", - "s3_endpoint_url": "https://localhost:443", - "s3_aws_access_key_id": "minioadmin", - "s3_aws_secret_access_key": "minioadmin", - "s3_region_name": "us-east-1", - "s3_verify": False, - }, + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": "https://localhost:443", + "s3_aws_access_key_id": "minioadmin", + "s3_aws_secret_access_key": "minioadmin", + "s3_region_name": "us-east-1", + "s3_verify": False, + }, ) with patch("asyncio.create_task"): @@ -948,9 +1090,9 @@ async def test_s3_verify_false_async_client(monkeypatch: pytest.MonkeyPatch): if hasattr(logger.async_httpx_client, "client"): httpx_client = logger.async_httpx_client.client if hasattr(httpx_client, "_verify"): - assert ( - httpx_client._verify is False - ), f"Expected async httpx client _verify=False, got {httpx_client._verify}" + assert httpx_client._verify is False, ( + f"Expected async httpx client _verify=False, got {httpx_client._verify}" + ) @pytest.mark.asyncio @@ -1017,9 +1159,7 @@ def patch_asyncio_create_task(): (True, True, None, None, ""), ], ) -def test_s3_object_key_prefix_combinations( - use_team_prefix, use_key_prefix, team_alias, key_alias, expected_prefix -): +def test_s3_object_key_prefix_combinations(use_team_prefix, use_key_prefix, team_alias, key_alias, expected_prefix): """ Validate correct S3 prefix composition for team alias + key alias combinations. """ @@ -1490,9 +1630,7 @@ def test_s3_callback_params_override_does_not_mutate_inputs(monkeypatch): logger = S3Logger(s3_callback_params_override=override) assert logger.s3_bucket_name == "resolved-bucket" assert override["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" - assert ( - litellm.s3_callback_params["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" - ) + assert litellm.s3_callback_params["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" def test_s3_callback_params_override_none_falls_back_to_global(monkeypatch): @@ -1520,9 +1658,7 @@ def _expected_content_md5(payload: dict) -> str: from litellm.litellm_core_utils.safe_json_dumps import safe_dumps json_string = safe_dumps(payload) - return base64.b64encode( - hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest() - ).decode() + return base64.b64encode(hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()).decode() def _require_non_security_md5(monkeypatch): @@ -1658,9 +1794,9 @@ def test_s3_server_side_encryption_read_from_callback_params(monkeypatch): litellm, "s3_callback_params", { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "aws:kms", - }, + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "aws:kms", + }, ) logger = S3Logger() assert logger.s3_server_side_encryption == "aws:kms" @@ -1789,10 +1925,10 @@ def test_s3_sse_kms_key_id_read_from_callback_params(monkeypatch): litellm, "s3_callback_params", { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "aws:kms", - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", - }, + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "aws:kms", + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", + }, ) logger = S3Logger() assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/test-key-id") @@ -1863,10 +1999,10 @@ def test_kms_key_id_dropped_when_algorithm_is_not_kms(monkeypatch): litellm, "s3_callback_params", { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "AES256", - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", - }, + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "AES256", + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", + }, ) logger = S3Logger() assert logger.s3_server_side_encryption == "AES256" @@ -1884,10 +2020,10 @@ def test_non_string_algorithm_is_dropped_and_valid_key_id_is_rescued(monkeypatch litellm, "s3_callback_params", { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": True, - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", - }, + "s3_bucket_name": "from-global", + "s3_server_side_encryption": True, + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", + }, ) logger = S3Logger() assert logger.s3_server_side_encryption == "aws:kms" @@ -1902,10 +2038,10 @@ def test_non_string_key_id_is_dropped_and_valid_algorithm_is_kept(monkeypatch): litellm, "s3_callback_params", { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "aws:kms", - "s3_sse_kms_key_id": 12345, - }, + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "aws:kms", + "s3_sse_kms_key_id": 12345, + }, ) logger = S3Logger() assert logger.s3_server_side_encryption == "aws:kms" @@ -2045,6 +2181,7 @@ async def test_download_signs_object_key_with_space_the_way_s3_does(): headers=call.kwargs["headers"], ) + _RESERVED_CHAR_KEYS = ( "2026-08-21/time-05-29-36_resp_bGl0ZWxsbTpjdXN0b20=.json", "session=logs/2026-08-21/time-05-29-36_abc.json", From e5da59336dacf9c69a07c1cb21dae2b9f4f828a4 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Thu, 10 Sep 2026 00:29:28 -0700 Subject: [PATCH 300/411] feat(prometheus): bucket latency by input sequence length (#40059) * feat(prometheus): bucket latency by input sequence length * style: format startup resolver call * fix(prometheus): handle unknown input lengths * fix(prometheus): preserve disabled custom input length labels * test(prometheus): seed startup snapshot in mocked logger test * fix(proxy): preserve database setting types during startup * fix(prometheus): distinguish missing usage and preserve config persistence Keep quoted database-storage config values intact for legacy persistence readers while using a local boolean for early callback discovery. Distinguish absent provider usage from an explicitly reported zero when labeling latency metrics. * fix(proxy): normalize input length flag from secret managers * fix(prometheus): isolate input buckets and preserve missing usage Keep built-in buckets on latency histograms, preserve unrelated custom labels, and classify raw incomplete usage and upstream total-only headers as unknown. Cover count conservation, failure callbacks, explicit zero, startup snapshots, and direct caller compatibility. Drop earlier branch budget changes. * fix(proxy): defer Prometheus alerting until stored settings load Reuse successful startup storage resolution and preserve callback deduplication across alerting reloads. * test(prometheus): restore input length flag between tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Revert "test(prometheus): restore input length flag between tests" This reverts commit f302e0b7bdad06e6c445bccdab0735aa8873ac39. * refactor(prometheus): make input length flag config/env only Drop the Admin UI General Settings row, the safe DB override entry, the startup reorder that loaded DB litellm_settings before Prometheus callbacks, and the alerting-only Prometheus path. The flag now behaves like prometheus_emit_stream_label: litellm_settings in config.yaml or an os.environ reference, applied on restart. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/integrations/prometheus.py | 57 ++- litellm/types/integrations/prometheus.py | 40 +- .../test_prometheus_client_ip_user_agent.py | 1 + ..._prometheus_input_sequence_length_label.py | 428 ++++++++++++++++++ 5 files changed, 518 insertions(+), 9 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_input_sequence_length_label.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 5a461801b62..2a85a3d4200 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -475,6 +475,7 @@ prometheus_metrics_config: Optional[List] = None prometheus_exclude_metrics: Optional[List[str]] = None prometheus_exclude_labels: Optional[List[str]] = None prometheus_emit_stream_label: bool = False +prometheus_emit_input_sequence_length_label: bool = False prometheus_deployment_and_latency_caller_identity: Literal[ "api_key_alias", "user_email", diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 6766d246894..540ce6738fc 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -246,6 +246,7 @@ class PrometheusLogger(CustomLogger): # logger so toggling these flags only takes effect after a # restart, keeping init-time and runtime label sets in sync. self._cached_metric_labels: dict[str, list[str]] = {} + self._emit_input_sequence_length_label = litellm.prometheus_emit_input_sequence_length_label is True _custom_buckets: Final = litellm.prometheus_latency_buckets self.latency_buckets = tuple(_custom_buckets) if _custom_buckets is not None else LATENCY_BUCKETS @@ -1522,6 +1523,11 @@ class PrometheusLogger(CustomLogger): # 2. Pyright does not allow us to run isinstance(standard_logging_payload, StandardLoggingPayload) <- this would be ideal enum_values=enum_values, label_context=label_context, + input_sequence_length=( + self._get_input_sequence_length(standard_logging_payload, kwargs, response_obj) + if self._emit_input_sequence_length_label + else None + ), ) # set x-ratelimit headers @@ -2192,6 +2198,36 @@ class PrometheusLogger(CustomLogger): ) self.litellm_remaining_api_key_tokens_for_model.labels(**tokens_labels).set(remaining_tokens) + @staticmethod + def _get_input_sequence_length( + standard_logging_payload: StandardLoggingPayload, + kwargs: Mapping[str, object], + response_obj: object, + ) -> str: + prompt_tokens: Final = standard_logging_payload.get("prompt_tokens") + if prompt_tokens: + return get_input_sequence_length_bucket(prompt_tokens) + combined_usage: Final = kwargs.get("combined_usage_object") + if ( + combined_usage is not None + and getattr(kwargs.get("_litellm_upstream_reported_usage"), "total_tokens", None) is not None + ): + return get_input_sequence_length_bucket(None) + reported_usage: Final = ( + response_obj.get("usage") if isinstance(response_obj, dict) else getattr(response_obj, "usage", None) + ) + if reported_usage is None and combined_usage is None: + return get_input_sequence_length_bucket(None) + usage_metadata: Final = standard_logging_payload["metadata"].get("usage_object") + if isinstance(usage_metadata, Mapping): + return get_input_sequence_length_bucket(usage_metadata.get("prompt_tokens")) + if combined_usage is None and isinstance(response_obj, dict): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + normalized_usage: Final[Mapping[str, object]] = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj) + return get_input_sequence_length_bucket(normalized_usage.get("prompt_tokens")) + return get_input_sequence_length_bucket(prompt_tokens) + def _set_latency_metrics( self, kwargs: dict, @@ -2202,7 +2238,16 @@ class PrometheusLogger(CustomLogger): user_api_team_alias: str | None, enum_values: UserAPIKeyLabelValues, label_context: PrometheusLabelFactoryContext | None = None, + input_sequence_length: str | None = None, ): + latency_enum_values: Final = ( + replace(enum_values, input_sequence_length=input_sequence_length) + if input_sequence_length is not None + else enum_values + ) + latency_label_context: Final = ( + PrometheusLabelFactoryContext(latency_enum_values) if input_sequence_length is not None else label_context + ) # latency metrics end_time: Final[datetime] = kwargs.get("end_time") or datetime.now() start_time: Final[datetime | None] = kwargs.get("start_time") @@ -2220,8 +2265,8 @@ class PrometheusLogger(CustomLogger): supported_enum_labels=self.get_labels_for_metric( metric_name="litellm_llm_api_time_to_first_token_metric" ), - enum_values=enum_values, - label_context=label_context, + enum_values=latency_enum_values, + label_context=latency_label_context, ) self.litellm_llm_api_time_to_first_token_metric.labels(**_ttft_labels).observe(time_to_first_token_seconds) self._track_end_user_metric_series( @@ -2241,8 +2286,8 @@ class PrometheusLogger(CustomLogger): if api_call_total_time_seconds is not None: _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_llm_api_latency_metric"), - enum_values=enum_values, - label_context=label_context, + enum_values=latency_enum_values, + label_context=latency_label_context, ) self.litellm_llm_api_latency_metric.labels(**_labels).observe(api_call_total_time_seconds) self._track_end_user_metric_series( @@ -2272,8 +2317,8 @@ class PrometheusLogger(CustomLogger): ) _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_total_latency_metric"), - enum_values=enum_values, - label_context=label_context, + enum_values=latency_enum_values, + label_context=latency_label_context, ) self.litellm_request_total_latency_metric.labels(**_labels).observe(_observed_total_time_seconds) self._track_end_user_metric_series( diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 8498b6f6d00..a024581f600 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -154,6 +154,22 @@ LATENCY_BUCKETS: Final = ( float("inf"), ) +UNKNOWN_INPUT_SEQUENCE_LENGTH: Final = "unknown" +INPUT_SEQUENCE_LENGTH_BUCKETS: Final = ( + (1_000, "0-1k"), + (4_000, "1k-4k"), + (16_000, "4k-16k"), + (64_000, "16k-64k"), + (float("inf"), "64k+"), +) + + +def get_input_sequence_length_bucket(prompt_tokens: object) -> str: + if not isinstance(prompt_tokens, int) or isinstance(prompt_tokens, bool) or prompt_tokens < 0: + return UNKNOWN_INPUT_SEQUENCE_LENGTH + return next(label for upper, label in INPUT_SEQUENCE_LENGTH_BUCKETS if prompt_tokens < upper) + + # Batch jobs can run for minutes to hours; buckets span 1 min → 24 h. BATCH_DURATION_BUCKETS: Final = ( 60.0, @@ -205,6 +221,7 @@ class UserAPIKeyLabelNames(Enum): MCP_TOOL_NAME = "mcp_tool_name" MCP_SERVER_NAME = "mcp_server_name" SERVICE_TIER = "service_tier" + INPUT_SEQUENCE_LENGTH = "input_sequence_length" DEFINED_PROMETHEUS_METRICS = Literal[ @@ -857,6 +874,13 @@ class PrometheusMetricLabels: "litellm_images_generated_metric", } ) + _input_sequence_length_metrics: ClassVar[frozenset[str]] = frozenset( + { + "litellm_llm_api_latency_metric", + "litellm_llm_api_time_to_first_token_metric", + "litellm_request_total_latency_metric", + } + ) # Managed batch metrics _batch_user_labels = [ UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, @@ -955,14 +979,23 @@ class PrometheusMetricLabels: custom_labels.append(label) if label_name in PrometheusMetricLabels._org_label_metrics: - for label in [ + for label in ( UserAPIKeyLabelNames.ORG_ID.value, UserAPIKeyLabelNames.ORG_ALIAS.value, - ]: + ): if label not in default_labels and label not in custom_labels: custom_labels.append(label) - return default_labels + custom_labels + input_sequence_length_labels: Final = ( + (UserAPIKeyLabelNames.INPUT_SEQUENCE_LENGTH.value,) + if ( + label_name in PrometheusMetricLabels._input_sequence_length_metrics + and litellm.prometheus_emit_input_sequence_length_label is True + and UserAPIKeyLabelNames.INPUT_SEQUENCE_LENGTH.value not in custom_labels + ) + else () + ) + return [*default_labels, *custom_labels, *input_sequence_length_labels] _USER_API_KEY_LABEL_VALUE_INIT_ALIASES: Final[Mapping[str, str]] = MappingProxyType( @@ -1015,6 +1048,7 @@ class UserAPIKeyLabelValues: mcp_tool_name: str | None = None mcp_server_name: str | None = None service_tier: str | None = None + input_sequence_length: str | None = None # Added for test compatibility. def __init__(self, **kwargs: Any) -> None: diff --git a/tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py b/tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py index ea661d2ea78..004ac4dbffb 100644 --- a/tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py +++ b/tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py @@ -84,6 +84,7 @@ async def test_async_post_call_success_hook_includes_client_ip_user_agent(): "litellm.integrations.prometheus.PrometheusLogger.__init__", return_value=None ): logger = PrometheusLogger() + logger._emit_input_sequence_length_label = False logger.litellm_proxy_total_requests_metric = MagicMock() logger.get_labels_for_metric = MagicMock( return_value=["client_ip", "user_agent"] diff --git a/tests/test_litellm/integrations/test_prometheus_input_sequence_length_label.py b/tests/test_litellm/integrations/test_prometheus_input_sequence_length_label.py new file mode 100644 index 00000000000..bc922061544 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_input_sequence_length_label.py @@ -0,0 +1,428 @@ +import asyncio +import datetime +from collections.abc import Mapping +from copy import deepcopy +from typing import Final, cast + +import pytest +from prometheus_client import REGISTRY +from prometheus_client.samples import Sample + +import litellm +from litellm.integrations.prometheus import PrometheusLogger +from litellm.types.integrations.prometheus import ( + PrometheusMetricLabels, + UserAPIKeyLabelNames, + UserAPIKeyLabelValues, + get_input_sequence_length_bucket, +) +from litellm.types.utils import StandardLoggingPayload + +LATENCY_METRICS: Final = ( + "litellm_llm_api_latency_metric", + "litellm_llm_api_time_to_first_token_metric", + "litellm_request_total_latency_metric", +) +FLAG: Final = "prometheus_emit_input_sequence_length_label" + + +def _clear_prometheus_registry() -> None: + for collector in tuple(REGISTRY._collector_to_names): # pyright: ignore[reportPrivateUsage] # test registry reset + REGISTRY.unregister(collector) + + +@pytest.fixture(autouse=True) +def isolated_registry(monkeypatch: pytest.MonkeyPatch): + _clear_prometheus_registry() + monkeypatch.setattr(litellm, FLAG, False) + yield + _clear_prometheus_registry() + + +@pytest.mark.parametrize("metric", LATENCY_METRICS) +def test_input_sequence_length_label_is_opt_in(monkeypatch: pytest.MonkeyPatch, metric: str): + assert UserAPIKeyLabelNames.INPUT_SEQUENCE_LENGTH.value not in PrometheusMetricLabels.get_labels(metric) + + monkeypatch.setattr(litellm, FLAG, True) + assert UserAPIKeyLabelNames.INPUT_SEQUENCE_LENGTH.value in PrometheusMetricLabels.get_labels(metric) + + +def test_input_sequence_length_label_stays_off_non_latency_metrics(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, FLAG, True) + assert UserAPIKeyLabelNames.INPUT_SEQUENCE_LENGTH.value not in PrometheusMetricLabels.get_labels( + "litellm_proxy_total_requests_metric" + ) + + +@pytest.mark.parametrize( + "prompt_tokens, expected", + [ + (None, "unknown"), + (0, "0-1k"), + (999, "0-1k"), + (1_000, "1k-4k"), + (3_999, "1k-4k"), + (4_000, "4k-16k"), + (15_999, "4k-16k"), + (16_000, "16k-64k"), + (63_999, "16k-64k"), + (64_000, "64k+"), + (10_000_000, "64k+"), + (-1, "unknown"), + ], +) +def test_input_sequence_length_bucket_boundaries(prompt_tokens: int | None, expected: str): + assert get_input_sequence_length_bucket(prompt_tokens) == expected + + +def test_user_api_key_label_values_carries_input_sequence_length(): + values: Final = UserAPIKeyLabelValues(input_sequence_length="4k-16k") + + assert values.input_sequence_length == "4k-16k" + assert values.model_dump()["input_sequence_length"] == "4k-16k" + + +def _assert_latency_metrics(expected: str | None, stream: bool = True, queue_time: float = 0) -> None: + samples: Final = tuple(sample for metric in REGISTRY.collect() for sample in metric.samples) + for metric, duration in zip(LATENCY_METRICS, (2, 1, 3 + queue_time)): + counts: Final = tuple(sample for sample in samples if sample.name == f"{metric}_count") + sums: Final = tuple(sample for sample in samples if sample.name == f"{metric}_sum") + buckets: Final = tuple(sample for sample in samples if sample.name == f"{metric}_bucket") + if not stream and metric == "litellm_llm_api_time_to_first_token_metric": + assert not counts and not sums and not buckets + continue + assert len(counts) == len(sums) == 1 + assert counts[0].value == 1 + assert sums[0].value == pytest.approx(duration) + assert buckets and any(sample.labels["le"] == "+Inf" for sample in buckets) + assert all(sample.value == int(float(sample.labels["le"]) >= duration) for sample in buckets) + assert all(sample.labels.get("input_sequence_length") == expected for sample in (*counts, *sums, *buckets)) + + +def _non_target_samples() -> tuple[Sample, ...]: + return tuple( + sample + for metric in REGISTRY.collect() + if metric.name not in LATENCY_METRICS + for sample in metric.samples + if "input_sequence_length" in sample.labels and not sample.name.endswith("_created") + ) + + +def _standard_logging_payload(now: datetime.datetime, prompt_tokens: int) -> StandardLoggingPayload: + return cast( + StandardLoggingPayload, + { + "id": "t", + "call_type": "completion", + "response_cost": 0.001, + "status": "success", + "total_tokens": prompt_tokens + 20, + "prompt_tokens": prompt_tokens, + "completion_tokens": 20, + "startTime": now - datetime.timedelta(seconds=3), + "endTime": now, + "completionStartTime": now - datetime.timedelta(seconds=1), + "model": "gpt-4o-mini", + "model_id": "model-123", + "model_group": "gpt-4o-mini", + "api_base": "https://api.openai.com", + "custom_llm_provider": "openai", + "request_tags": [], + "stream": True, + "metadata": { + "user_api_key_hash": "h", + "user_api_key_alias": "a", + "user_api_key_team_id": "t", + "user_api_key_team_alias": "ta", + "user_api_key_user_id": "u", + "user_api_key_user_email": "e@x.com", + "user_api_key_org_id": None, + "user_api_key_org_alias": None, + "requester_metadata": None, + "user_api_key_end_user_id": None, + "usage_object": None, + }, + "hidden_params": {"litellm_overhead_time_ms": None, "additional_headers": None}, + }, + ) + + +def _success_kwargs( + now: datetime.datetime, prompt_tokens: int, requester_metadata: Mapping[str, object] | None = None +) -> Mapping[str, object]: + payload: Final = _standard_logging_payload(now, prompt_tokens) + return { + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {}}, + "standard_logging_object": { + **payload, + "metadata": {**payload["metadata"], "requester_metadata": requester_metadata}, + }, + "stream": True, + "start_time": now - datetime.timedelta(seconds=3), + "api_call_start_time": now - datetime.timedelta(seconds=2), + "completion_start_time": now - datetime.timedelta(seconds=1), + "end_time": now, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flag_at_request_time", (True, False)) +async def test_logger_emits_bucket_from_its_startup_label_set( + monkeypatch: pytest.MonkeyPatch, flag_at_request_time: bool +): + now: Final = datetime.datetime.now() + monkeypatch.setattr(litellm, FLAG, True) + logger: Final = PrometheusLogger() + monkeypatch.setattr(litellm, FLAG, flag_at_request_time) + + await logger.async_log_success_event(dict(_success_kwargs(now, prompt_tokens=4_000)), None, now, now) + + _assert_latency_metrics("4k-16k") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("response", "combined_usage", "expected"), + ( + ({"id": "moderation", "results": []}, None, "unknown"), + ({"usage": None}, None, "unknown"), + ({"usage": {}}, None, "unknown"), + ({"usage": {"completion_tokens": 3}}, None, "unknown"), + ({"usage": {"total_tokens": 5}}, None, "unknown"), + ({"usage": {"prompt_tokens": 0}}, None, "0-1k"), + ({"usage": {"prompt_tokens": 4_000}}, None, "4k-16k"), + ({"usage": {"input_tokens": 0, "output_tokens": 3, "total_tokens": 3}}, None, "0-1k"), + ({"usage": {"input_tokens": 4_000, "output_tokens": 3, "total_tokens": 4_003}}, None, "4k-16k"), + (litellm.ModelResponse(usage=litellm.Usage(prompt_tokens=0)), None, "0-1k"), + (None, litellm.Usage(prompt_tokens=0), "0-1k"), + ), +) +@pytest.mark.parametrize("include_usage_metadata", (True, False)) +async def test_logger_distinguishes_missing_usage_from_reported_zero( + monkeypatch: pytest.MonkeyPatch, + response: object, + combined_usage: object, + expected: str, + include_usage_metadata: bool, +): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + now: Final = datetime.datetime.now() + monkeypatch.setattr(litellm, FLAG, True) + logger: Final = PrometheusLogger() + usage: Final = StandardLoggingPayloadSetup.get_usage_as_dict( + response_obj=response if isinstance(response, dict) else None + ) + + payload: Final = _standard_logging_payload(now, usage.get("prompt_tokens", 0)) + await logger.async_log_success_event( + { + **_success_kwargs(now, prompt_tokens=usage.get("prompt_tokens", 0)), + "combined_usage_object": combined_usage, + "standard_logging_object": { + **payload, + "metadata": {**payload["metadata"], "usage_object": usage if include_usage_metadata else None}, + }, + }, + response, + now, + now, + ) + + _assert_latency_metrics(expected) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("total_tokens", (None, 0, 5_000)) +@pytest.mark.parametrize("prompt_tokens", (0, 4_000)) +async def test_upstream_total_only_usage_has_unknown_input_length( + monkeypatch: pytest.MonkeyPatch, total_tokens: int | None, prompt_tokens: int +): + import httpx + + from litellm.litellm_core_utils.litellm_logging import Logging, StandardLoggingPayloadSetup + from litellm.proxy.pass_through_endpoints.upstream_usage_headers import apply_upstream_reported_usage + + now: Final = datetime.datetime.now() + monkeypatch.setattr(litellm, FLAG, True) + logger: Final = PrometheusLogger() + logging_obj: Final = Logging( + model="gpt-4o-mini", + messages=[], + stream=True, + call_type="pass_through_endpoint", + start_time=now, + litellm_call_id="test-call-id", + function_id="1", + ) + headers: Final = httpx.Headers( + { + "x-litellm-response-cost": "0.001", + **({"x-litellm-total-tokens": str(total_tokens)} if total_tokens is not None else {}), + } + ) + reported: Final = apply_upstream_reported_usage(logging_obj=logging_obj, headers=headers) + assert reported is not None + combined_usage: Final = logging_obj.model_call_details.get("combined_usage_object") + response: Final = {"usage": {"prompt_tokens": prompt_tokens}} + usage: Final = StandardLoggingPayloadSetup.get_usage_as_dict(response, combined_usage) + payload: Final = _standard_logging_payload(now, usage.get("prompt_tokens", 0)) + + await logger.async_log_success_event( + { + **logging_obj.model_call_details, + **_success_kwargs(now, usage.get("prompt_tokens", 0)), + "standard_logging_object": {**payload, "metadata": {**payload["metadata"], "usage_object": usage}}, + }, + response, + now, + now, + ) + + _assert_latency_metrics("unknown" if total_tokens is not None else get_input_sequence_length_bucket(prompt_tokens)) + + +@pytest.mark.asyncio +async def test_logger_built_with_flag_off_emits_no_bucket_label(monkeypatch: pytest.MonkeyPatch): + now: Final = datetime.datetime.now() + logger: Final = PrometheusLogger() + monkeypatch.setattr(litellm, FLAG, True) + + await logger.async_log_success_event(dict(_success_kwargs(now, prompt_tokens=4_000)), None, now, now) + + _assert_latency_metrics(None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flag_at_startup", (True, False)) +@pytest.mark.parametrize("stream", (True, False)) +@pytest.mark.parametrize( + "metadata", + ( + None, + {}, + {"input_sequence_length": None}, + {"input_sequence_length": False}, + {"input_sequence_length": True}, + {"input_sequence_length": 0}, + {"input_sequence_length": []}, + {"input_sequence_length": {}}, + {"input_sequence_length": ""}, + {"input_sequence_length": "from-metadata"}, + ), +) +async def test_custom_input_length_label_is_scoped_to_target_histograms( + monkeypatch: pytest.MonkeyPatch, flag_at_startup: bool, stream: bool, metadata: Mapping[str, object] | None +): + now: Final = datetime.datetime.now() + monkeypatch.setattr(litellm, "custom_prometheus_metadata_labels", ["input_sequence_length"]) + kwargs: Final = { + **_success_kwargs(now, prompt_tokens=4_000, requester_metadata=metadata), + "stream": stream, + "litellm_params": {"metadata": {"queue_time_seconds": 0.25}}, + } + original_kwargs: Final = deepcopy(kwargs) + baseline_logger: Final = PrometheusLogger() + await baseline_logger.async_log_success_event(kwargs, None, now, now) + baseline_samples: Final = _non_target_samples() + _clear_prometheus_registry() + monkeypatch.setattr(litellm, FLAG, flag_at_startup) + logger: Final = PrometheusLogger() + monkeypatch.setattr(litellm, FLAG, not flag_at_startup) + + await logger.async_log_success_event(kwargs, None, now, now) + + assert kwargs == original_kwargs + custom_value: Final = (metadata or {}).get("input_sequence_length") + expected: Final = custom_value if isinstance(custom_value, str) else ("4k-16k" if flag_at_startup else "None") + _assert_latency_metrics(expected, stream=stream, queue_time=0.25) + assert all(logger.get_labels_for_metric(metric).count("input_sequence_length") == 1 for metric in LATENCY_METRICS) + non_target_samples: Final = _non_target_samples() + assert { + "litellm_requests_metric_total", + "litellm_spend_metric_total", + "litellm_total_tokens_metric_total", + "litellm_request_queue_time_seconds_count", + "litellm_deployment_success_responses_total", + }.issubset({sample.name for sample in non_target_samples}) + assert non_target_samples == baseline_samples + queue_sum: Final = tuple( + sample for sample in non_target_samples if sample.name == "litellm_request_queue_time_seconds_sum" + ) + assert len(queue_sum) == 1 and queue_sum[0].value == 0.25 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", (True, False)) +async def test_concurrent_requests_keep_independent_buckets(monkeypatch: pytest.MonkeyPatch, enabled: bool): + now: Final = datetime.datetime.now() + monkeypatch.setattr(litellm, FLAG, enabled) + logger: Final = PrometheusLogger() + cases: Final = ( + (None, "unknown"), + (0, "0-1k"), + (1_000, "1k-4k"), + (4_000, "4k-16k"), + (16_000, "16k-64k"), + (64_000, "64k+"), + ) + calls: Final = tuple( + ( + {**_success_kwargs(now, prompt_tokens=tokens or 0), "stream": stream}, + {"usage": {"prompt_tokens": tokens}} if tokens is not None else None, + ) + for tokens, _ in cases + for stream in (True, False) + for _ in range(2) + ) + original_calls: Final = deepcopy(calls) + + await asyncio.gather(*(logger.async_log_success_event(kwargs, response, now, now) for kwargs, response in calls)) + + assert calls == original_calls + samples: Final = tuple(sample for metric in REGISTRY.collect() for sample in metric.samples) + for metric, duration in zip(LATENCY_METRICS, (2, 1, 3)): + expected_count: Final = 2 if metric == "litellm_llm_api_time_to_first_token_metric" else 4 + counts: Final = tuple(sample for sample in samples if sample.name == f"{metric}_count") + sums: Final = tuple(sample for sample in samples if sample.name == f"{metric}_sum") + buckets: Final = tuple(sample for sample in samples if sample.name == f"{metric}_bucket") + expected: Final = ( + {bucket: expected_count for _, bucket in cases} if enabled else {None: expected_count * len(cases)} + ) + assert len(counts) == len(sums) == len(expected) + assert {sample.labels.get("input_sequence_length"): sample.value for sample in counts} == expected + assert {sample.labels.get("input_sequence_length"): sample.value for sample in sums} == { + bucket: count * duration for bucket, count in expected.items() + } + assert sum(sample.value for sample in buckets if sample.labels["le"] == "+Inf") == expected_count * len(cases) + assert all( + sample.value + == expected[sample.labels.get("input_sequence_length")] * int(float(sample.labels["le"]) >= duration) + for sample in buckets + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", (True, False)) +async def test_failed_request_does_not_observe_latency(monkeypatch: pytest.MonkeyPatch, enabled: bool): + now: Final = datetime.datetime.now() + monkeypatch.setattr(litellm, FLAG, enabled) + monkeypatch.setattr(litellm, "custom_prometheus_metadata_labels", ["input_sequence_length"]) + logger: Final = PrometheusLogger() + kwargs: Final = { + **_success_kwargs(now, prompt_tokens=4_000), + "standard_logging_object": {**_standard_logging_payload(now, 4_000), "status": "failure"}, + "exception": RuntimeError("upstream request failed"), + } + + await logger.async_log_failure_event(kwargs, None, now, now) + + samples: Final = tuple(sample for metric in REGISTRY.collect() for sample in metric.samples) + assert not any(sample.name.startswith(LATENCY_METRICS) for sample in samples) + for metric in ("litellm_llm_api_failed_requests_metric_total", "litellm_deployment_failure_responses_total"): + counts: Final = tuple(sample for sample in samples if sample.name == metric) + assert len(counts) == 1 + assert counts[0].value == 1 + assert counts[0].labels["input_sequence_length"] == "None" From 0b898b47ca4f3713ccc38446f7146109e57ec11a Mon Sep 17 00:00:00 2001 From: Yinon Kahta Date: Mon, 7 Sep 2026 09:53:43 +0300 Subject: [PATCH 301/411] fix(http_handler): let put opt out of following redirects get and post already take follow_redirects. put built the request and sent it with the client default, so a caller uploading to a URL it did not choose had no way to refuse a redirect. Same plumbing as the other two methods. --- litellm/llms/custom_httpx/http_handler.py | 22 +++++-- .../llms/custom_httpx/test_http_handler.py | 65 +++++++++++++++++++ 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index e1f0fc9e7d3..12e515e19a8 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -751,7 +751,9 @@ class AsyncHTTPHandler: timeout: float | httpx.Timeout | None = None, stream: bool = False, content: _RequestContent | None = None, + follow_redirects: bool | None = None, ): + _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT try: if timeout is None: timeout = self.timeout @@ -769,22 +771,30 @@ class AsyncHTTPHandler: timeout=timeout, content=request_content, ) - response: Final = await self.client.send(req) + response: Final = await self.client.send(req, follow_redirects=_follow_redirects) response.raise_for_status() return response except (httpx.RemoteProtocolError, httpx.ConnectError): # Retry the request with a new session if there is a connection error new_client: Final = self.create_client(timeout=timeout, event_hooks=self.event_hooks) try: - return await self.single_connection_post_request( - url=url, - client=new_client, - data=data, + retry_data, retry_content = _prepare_request_data_and_content(data, content) + retry: Final = new_client.build_request( + "PUT", + url, + data=retry_data, json=json, params=params, headers=headers, - stream=stream, + timeout=timeout, + content=retry_content, ) + retried: Final = await new_client.send(retry, stream=stream, follow_redirects=_follow_redirects) + try: + retried.raise_for_status() + except httpx.HTTPStatusError as retried_error: + await _raise_masked_async_error(retried_error, stream) + return retried finally: await new_client.aclose() except httpx.TimeoutException as e: diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 9e64bfafa54..3d4ba264c1c 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1547,3 +1547,68 @@ def test_sync_force_ipv4_https_proxy_mount_uses_handler_ca_bundle( handler.close() assert response.text == "ok-tls" + + +@pytest.mark.asyncio +async def test_put_can_refuse_to_follow_a_redirect(): + """The client follows redirects by default; a caller uploading to a URL it did not choose must be able to opt out.""" + hops: list[str] = [] # mutable-ok: the fake transport records the paths it was asked for + + async def mock_handler(request: httpx.Request) -> httpx.Response: + hops.append(request.url.path) + if request.url.path == "/first": + return httpx.Response(302, request=request, headers={"location": "/second"}) + return httpx.Response(200, request=request) + + handler = AsyncHTTPHandler() + await handler.client.aclose() + handler.client = httpx.AsyncClient(transport=httpx.MockTransport(mock_handler), follow_redirects=True) + try: + followed = await handler.put("https://uploads.example/first", data=b"x") + assert followed.status_code == 200 + assert hops == ["/first", "/second"] + + hops.clear() + with pytest.raises(MaskedHTTPStatusError) as refused: + await handler.put("https://uploads.example/first", data=b"x", follow_redirects=False) + assert refused.value.status_code == 302 + assert hops == ["/first"] + finally: + await handler.close() + + +@pytest.mark.asyncio +async def test_a_retried_put_stays_a_put_and_still_refuses_redirects(): + """ + The connection-error retry used to resend as POST through a client that follows redirects. + + Storage answers a POST to a presigned PUT url with 403 or 405, so the batch looked + permanently rejected, and the redirect refusal the caller asked for was silently lost. + """ + attempts: list[tuple[str, str]] = [] # mutable-ok: the fake transports record what they were asked for + + async def refusing_transport(request: httpx.Request) -> httpx.Response: + attempts.append((request.method, request.url.path)) + raise httpx.ConnectError("connection reset", request=request) + + async def retry_transport(request: httpx.Request) -> httpx.Response: + attempts.append((request.method, request.url.path)) + if request.url.path == "/first": + return httpx.Response(302, request=request, headers={"location": "/second"}) + return httpx.Response(200, request=request) + + class HandlerWithFakeRetryClient(AsyncHTTPHandler): + def create_client(self, *args, **kwargs) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.MockTransport(retry_transport), follow_redirects=True) + + handler = HandlerWithFakeRetryClient() + await handler.client.aclose() + handler.client = httpx.AsyncClient(transport=httpx.MockTransport(refusing_transport)) + try: + with pytest.raises(MaskedHTTPStatusError) as refused: + await handler.put("https://uploads.example/first", data=b"x", follow_redirects=False) + + assert refused.value.status_code == 302 + assert attempts == [("PUT", "/first"), ("PUT", "/first")] + finally: + await handler.client.aclose() From 6d7b3a82ffc2078a8b7498c81a965ec217c74289 Mon Sep 17 00:00:00 2001 From: Yinon Kahta Date: Thu, 27 Aug 2026 13:05:57 +0300 Subject: [PATCH 302/411] feat(pointfive): add gzipped ndjson batch encoding --- litellm/integrations/pointfive/payload.py | 53 ++++++++++++ .../integrations/pointfive/test_payload.py | 80 +++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 litellm/integrations/pointfive/payload.py create mode 100644 tests/test_litellm/integrations/pointfive/test_payload.py diff --git a/litellm/integrations/pointfive/payload.py b/litellm/integrations/pointfive/payload.py new file mode 100644 index 00000000000..e3362eda4ee --- /dev/null +++ b/litellm/integrations/pointfive/payload.py @@ -0,0 +1,53 @@ +"""Turns buffered log records into the gzipped NDJSON objects that get uploaded.""" + +import gzip +from collections.abc import Iterator, Mapping, Sequence +from itertools import accumulate, groupby, islice +from typing import Final + +from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + +_NEWLINE_BYTES: Final = 1 + + +def serialize_records(records: Sequence[Mapping[str, object]]) -> tuple[str, ...]: + """Serialize each record to one JSON line.""" + return tuple(safe_dumps(record) for record in records) + + +def _encoded_size(line: str) -> int: + return len(line.encode("utf-8")) + _NEWLINE_BYTES + + +def _object_indices(sizes: Sequence[int], max_bytes: int) -> Iterator[int]: + """Number each line with the object it belongs to, opening a new one on overflow.""" + + def advance(state: tuple[int, int], size: int) -> tuple[int, int]: + index, used = state + return (index + 1, size) if used and used + size > max_bytes else (index, used + size) + + return (index for index, _ in islice(accumulate(sizes, advance, initial=(0, 0)), 1, None)) + + +def chunk_lines(lines: Sequence[str], max_bytes: int) -> tuple[tuple[str, ...], ...]: + """ + Group serialized lines into objects of at most ``max_bytes`` uncompressed. + + A line above the bound on its own still becomes its own object. A record cannot be + split, and holding it back would stall every record queued behind it. + """ + sizes: Final = tuple(_encoded_size(line) for line in lines) + numbered: Final = zip(_object_indices(sizes, max_bytes), lines, strict=True) + return tuple(tuple(line for _, line in group) for _, group in groupby(numbered, lambda pair: pair[0])) + + +async def encode_lines(lines: Sequence[str]) -> bytes: + """ + Join lines as NDJSON and gzip them off the event loop. + + An object can be several megabytes, and compressing that inline would block the + proxy for as long as it takes. + """ + compress: Final = asyncify(gzip.compress) + return await compress("\n".join(lines).encode("utf-8")) diff --git a/tests/test_litellm/integrations/pointfive/test_payload.py b/tests/test_litellm/integrations/pointfive/test_payload.py new file mode 100644 index 00000000000..d310941e396 --- /dev/null +++ b/tests/test_litellm/integrations/pointfive/test_payload.py @@ -0,0 +1,80 @@ +import gzip +import json + +import pytest + +from litellm.integrations.pointfive.payload import chunk_lines, encode_lines, serialize_records + +UNBOUNDED = 10_000_000 + + +def test_each_record_becomes_one_json_line(): + lines = serialize_records([{"id": "a"}, {"id": "b"}, {"id": "c"}]) + + assert len(lines) == 3 + assert [json.loads(line)["id"] for line in lines] == ["a", "b", "c"] + + +def test_non_serializable_values_do_not_raise(): + """An odd payload must not kill the flush.""" + lines = serialize_records([{"id": "a", "when": object()}]) + + assert json.loads(lines[0])["id"] == "a" + + +def test_records_that_fit_stay_in_one_object(): + lines = serialize_records([{"id": f"r{i}"} for i in range(50)]) + + assert chunk_lines(lines, UNBOUNDED) == (lines,) + + +def test_objects_are_capped_by_uncompressed_size(): + lines = serialize_records([{"id": f"r{i}", "blob": "x" * 100} for i in range(10)]) + line_bytes = len(lines[0].encode("utf-8")) + 1 + + chunks = chunk_lines(lines, line_bytes * 3) + + assert [len(chunk) for chunk in chunks] == [3, 3, 3, 1] + + +def test_oversized_single_record_is_sent_alone_not_stalled(): + """A record too big for the cap must still go out, or it blocks everything behind it.""" + lines = serialize_records([{"id": "small"}, {"id": "huge", "blob": "x" * 5000}, {"id": "small2"}]) + + chunks = chunk_lines(lines, 200) + + assert sum(len(chunk) for chunk in chunks) == 3 + huge = [chunk for chunk in chunks if any("huge" in line for line in chunk)] + assert len(huge) == 1 + assert len(huge[0]) == 1 + + +def test_no_records_produces_no_objects(): + assert chunk_lines((), UNBOUNDED) == () + + +def test_every_record_appears_exactly_once(): + lines = serialize_records([{"id": f"r{i}"} for i in range(37)]) + + chunks = chunk_lines(lines, len(lines[0]) * 4) + + assert [line for chunk in chunks for line in chunk] == list(lines) + + +@pytest.mark.asyncio +async def test_encode_lines_round_trips_through_gzip(): + lines = serialize_records([{"id": f"r{i}"} for i in range(5)]) + + encoded = await encode_lines(lines) + + assert encoded[:2] == b"\x1f\x8b" + assert gzip.decompress(encoded).decode("utf-8") == "\n".join(lines) + + +@pytest.mark.asyncio +async def test_encode_lines_compresses_repetitive_records(): + lines = serialize_records([{"id": f"r{i}", "model": "gpt-4o", "cost": 0.01} for i in range(200)]) + + encoded = await encode_lines(lines) + + assert len(encoded) < len(gzip.decompress(encoded)) / 2 From 85d10c8558adb4173aba86db47ff6ad757c4d6d5 Mon Sep 17 00:00:00 2001 From: Yinon Kahta Date: Thu, 27 Aug 2026 13:05:58 +0300 Subject: [PATCH 303/411] feat(pointfive): add presigned upload client --- .../integrations/pointfive/upload_client.py | 194 +++++++++ litellm/types/integrations/pointfive.py | 45 +++ .../pointfive/test_upload_client.py | 378 ++++++++++++++++++ 3 files changed, 617 insertions(+) create mode 100644 litellm/integrations/pointfive/upload_client.py create mode 100644 litellm/types/integrations/pointfive.py create mode 100644 tests/test_litellm/integrations/pointfive/test_upload_client.py diff --git a/litellm/integrations/pointfive/upload_client.py b/litellm/integrations/pointfive/upload_client.py new file mode 100644 index 00000000000..56ba6689017 --- /dev/null +++ b/litellm/integrations/pointfive/upload_client.py @@ -0,0 +1,194 @@ +""" +Uploads one batch to PointFive through a presigned URL. + +The proxy holds no cloud credentials. For every batch it asks the PointFive API for a +single-use presigned URL and PUTs the bytes there, so the same plugin runs unchanged on +AWS, GCP, Azure or on-prem. The server picks the object key, so the proxy never chooses +where its data lands. +""" + +import asyncio +from collections.abc import Awaitable, Callable +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import BaseModel, Field, ValidationError + +import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import SSRFError, validate_url +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.integrations.pointfive import ( + RETRYABLE_UPLOAD_STATUS_CODES, + PointFiveUploadFailure, + PointFiveUploadTarget, +) + +UPLOAD_KIND: Final = "LITELLM" +UPLOAD_URL_PATH: Final = "/upload-url" +PING_PATH: Final = "/ping" +PUT_HEADERS: Final = MappingProxyType({"Content-Type": "application/x-ndjson", "Content-Encoding": "gzip"}) + + +class _PresignRequest(BaseModel): + kind: str = UPLOAD_KIND + byte_count: int = Field(serialization_alias="byteCount") + + +class _PingRequest(BaseModel): + kind: str = UPLOAD_KIND + + +class _TargetPayload(BaseModel): + upload_url: str = Field(alias="uploadUrl") + object_key: str = Field(alias="objectKey") + + +class _ErrorPayload(BaseModel): + error: str = "" + + +class PointFiveUploadError(Exception): + """A batch could not be uploaded and the failure is worth retrying.""" + + +def _failure_for(response: httpx.Response, what: str) -> PointFiveUploadFailure: + detail: Final = f"{what} returned {response.status_code}" + reason: Final = _refusal_reason(response.text) + return PointFiveUploadFailure( + f"{detail}, {reason}" if reason else detail, + retryable=response.status_code in RETRYABLE_UPLOAD_STATUS_CODES, + ) + + +def _refusal_reason(body: str) -> str: + try: + return _ErrorPayload.model_validate_json(body).error + except ValidationError: + return "" + + +def _parse_target(body: str) -> PointFiveUploadTarget | PointFiveUploadFailure: + try: + target: Final = _TargetPayload.model_validate_json(body) + except ValidationError: + return PointFiveUploadFailure("pointfive api returned an unreadable body", retryable=False) + return PointFiveUploadTarget(upload_url=target.upload_url, object_key=target.object_key) + + +class PointFiveUploadClient: + """Presigns and uploads one batch at a time.""" + + def __init__( + self, + api_key: str, + api_url: str, + http_client: AsyncHTTPHandler, + max_retries: int, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + validate_upload_url: Callable[[str], tuple[str, str]] = validate_url, + ) -> None: + self.api_key: Final = api_key + self.api_url: Final = api_url.rstrip("/") + self.http_client: Final = http_client + self.max_retries: Final = max_retries + self.sleep: Final = sleep + self.validate_upload_url: Final = validate_upload_url + + async def upload(self, body: bytes) -> str | PointFiveUploadFailure: + """ + Upload one gzipped batch, returning the object key it landed at. + + Every attempt presigns again, so a retry never reuses a URL that has expired or + has already been consumed. + """ + for attempt in range(self.max_retries): + match await self._upload_once(body): + case PointFiveUploadFailure(retryable=True) as failure: + if attempt + 1 >= self.max_retries: + return PointFiveUploadFailure( + f"{failure.detail}, gave up after {self.max_retries} attempts", retryable=True + ) + await self.sleep(float(1 << attempt)) + case outcome: + return outcome + return PointFiveUploadFailure("max_upload_retries must be at least 1", retryable=False) + + async def _upload_once(self, body: bytes) -> str | PointFiveUploadFailure: + target: Final = await self._presign(len(body)) + if isinstance(target, PointFiveUploadFailure): + return target + + rejection: Final = await self._put(target, body) + if rejection is not None: + return rejection + + verbose_logger.debug("pointfive: uploaded %s gzipped bytes to %s", len(body), target.object_key) + return target.object_key + + async def ping(self) -> PointFiveUploadFailure | None: + """Report that the proxy is alive when it has nothing to upload.""" + body: Final = await self._post(PING_PATH, _PingRequest()) + if isinstance(body, PointFiveUploadFailure): + return body + return None + + async def _presign(self, byte_count: int) -> PointFiveUploadTarget | PointFiveUploadFailure: + """Ask the PointFive API for a presigned URL sized to this batch.""" + body: Final = await self._post(UPLOAD_URL_PATH, _PresignRequest(byte_count=byte_count)) + if isinstance(body, PointFiveUploadFailure): + return body + return _parse_target(body) + + async def _post(self, path: str, request: BaseModel) -> str | PointFiveUploadFailure: + """POST one JSON request to the PointFive ingestion API and return its raw body.""" + try: + response: Final = await self.http_client.post( + self.api_url + path, + json=request.model_dump(by_alias=True), + headers={ # mutable-ok: AsyncHTTPHandler.post types headers as dict + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + ) + except httpx.HTTPStatusError as e: + return _failure_for(e.response, "pointfive api") + except Exception as e: # noqa: BLE001 # a transport fault is worth another attempt + return PointFiveUploadFailure(f"pointfive api unreachable: {type(e).__name__}", retryable=True) + return response.text + + async def _put(self, target: PointFiveUploadTarget, body: bytes) -> PointFiveUploadFailure | None: + """ + PUT the batch to the presigned URL, which carries its own authorization. + + The server chose that URL, so it is treated like any other externally supplied + destination: the host is checked against blocked networks before connecting, and + a redirect is refused rather than followed. A presigned URL never legitimately + redirects, and following one would let a compromised endpoint point the proxy at + an internal service. + """ + destination: Final = self._destination(target.upload_url) + if isinstance(destination, PointFiveUploadFailure): + return destination + url, host = destination + headers: Final = dict(PUT_HEADERS, Host=host) if host else dict(PUT_HEADERS) # mutable-ok: put wants dict + try: + await self.http_client.put(url, data=body, headers=headers, follow_redirects=False) + except httpx.HTTPStatusError as e: + if e.response.is_redirect: + return PointFiveUploadFailure( + f"presigned upload redirected with {e.response.status_code}, refusing to follow", retryable=False + ) + return _failure_for(e.response, "presigned upload") + except Exception as e: # noqa: BLE001 # a transport fault is worth another attempt + return PointFiveUploadFailure(f"presigned upload unreachable: {type(e).__name__}", retryable=True) + return None + + def _destination(self, upload_url: str) -> tuple[str, str | None] | PointFiveUploadFailure: + if not getattr(litellm, "user_url_validation", True): + return upload_url, None + try: + return self.validate_upload_url(upload_url) + except SSRFError as e: + return PointFiveUploadFailure(f"presigned upload url refused: {e}", retryable=False) diff --git a/litellm/types/integrations/pointfive.py b/litellm/types/integrations/pointfive.py new file mode 100644 index 00000000000..9ff17393ecd --- /dev/null +++ b/litellm/types/integrations/pointfive.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass +from typing import Final + +from pydantic import Field + +from litellm.types.integrations.custom_logger import StandardCustomLoggerInitParams + +RETRYABLE_UPLOAD_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504}) + +DEFAULT_API_URL: Final = "https://api.pointfive.co/api/v1/ingestion" + + +class PointFiveInitParams(StandardCustomLoggerInitParams): + """ + Params for initializing a PointFive logger on litellm. + + Defaults trade freshness for fewer, larger uploads: every flush becomes one object, so + the interval is minutes rather than seconds. ``batch_size`` also bounds how much a busy + proxy holds in memory between flushes, so it stays modest. ``max_batch_bytes`` bounds + how much a single object may hold, which matters most when message logging is left on, + since an unredacted payload is orders of magnitude larger than a redacted one. + """ + + api_key: str | None = None + api_url: str | None = None + batch_size: int = Field(default=1_000, gt=0) + flush_interval: int = Field(default=300, gt=0) + max_batch_bytes: int = Field(default=8 * 1024 * 1024, gt=0) + max_upload_retries: int = Field(default=3, ge=1) + + +@dataclass(frozen=True, slots=True) +class PointFiveUploadTarget: + """A single-use presigned destination for one batch, issued by the PointFive API.""" + + upload_url: str + object_key: str + + +@dataclass(frozen=True, slots=True) +class PointFiveUploadFailure: + """Why a batch could not be uploaded, and whether a later attempt could still succeed.""" + + detail: str + retryable: bool diff --git a/tests/test_litellm/integrations/pointfive/test_upload_client.py b/tests/test_litellm/integrations/pointfive/test_upload_client.py new file mode 100644 index 00000000000..50ef085386d --- /dev/null +++ b/tests/test_litellm/integrations/pointfive/test_upload_client.py @@ -0,0 +1,378 @@ +import json +from collections.abc import Sequence + +import httpx +import pytest + +import litellm +from litellm.integrations.pointfive.upload_client import PointFiveUploadClient +from litellm.litellm_core_utils.url_utils import validate_url +from litellm.types.integrations.pointfive import PointFiveUploadFailure + +API_URL = "https://api.pointfive.co/api/v1/ingestion" +UPLOAD_URL = "https://uploads.example.invalid/some/object.ndjson.gz?signature=sig" +OBJECT_KEY = "some/object.ndjson.gz" +BODY = b"gzipped-bytes" + + +def _presigned(status_code: int = 200) -> httpx.Response: + return _response( + status_code, {"uploadUrl": UPLOAD_URL, "objectKey": OBJECT_KEY, "expiresAt": "2026-08-25T14:35:00Z"} + ) + + +def _response(status_code: int, payload: object) -> httpx.Response: + return httpx.Response(status_code, text=json.dumps(payload)) + + +def _refused(status_code: int, error: str) -> httpx.Response: + """The body PointFive sends with every refusal.""" + return _response(status_code, {"success": False, "error": error}) + + +def _accepted() -> httpx.Response: + return httpx.Response(200, text="") + + +def _no_content() -> httpx.Response: + return httpx.Response(204, text="") + + +class FakeHTTPClient: + """ + Stands in for AsyncHTTPHandler, including its habit of raising on error statuses. + + Scripted results are consumed in order, and the last one repeats, so a test that + cares about a single behaviour passes a single result. + """ + + def __init__( + self, + presign: Sequence[httpx.Response | Exception] | None = None, + put: Sequence[httpx.Response | Exception] | None = None, + ) -> None: + self.presign = list(presign) if presign else [_presigned()] # mutable-ok: results are consumed by popping + self.put_results = list(put) if put else [_accepted()] # mutable-ok: results are consumed by popping + self.presign_calls: list[dict] = [] + self.put_calls: list[dict] = [] + + async def post(self, url, json=None, headers=None, **_): + self.presign_calls.append({"url": url, "json": json, "headers": headers or {}}) + return _next_result(self.presign, url) + + async def put(self, url, data=None, headers=None, follow_redirects=None, **_): + self.put_calls.append( + {"url": url, "data": data, "headers": headers or {}, "follow_redirects": follow_redirects} + ) + return _next_result(self.put_results, url) + + +def _next_result(results: list, url: str) -> httpx.Response: + result = results.pop(0) if len(results) > 1 else results[0] + if isinstance(result, Exception): + raise result + if result.status_code >= 300: + request = httpx.Request("POST", url) + raise httpx.HTTPStatusError( + "boom", + request=request, + response=httpx.Response(result.status_code, text=result.text, headers=result.headers), + ) + return result + + +async def _no_backoff(_seconds: float) -> None: + return None + + +def _trusting_validator(url: str) -> tuple[str, str]: + """Stands in for validate_url so the fixture hosts need no DNS; the SSRF tests use the real one.""" + return url, httpx.URL(url).host + + +def _client( + http_client: FakeHTTPClient, + max_retries: int = 3, + api_url: str = API_URL, + validate_upload_url=_trusting_validator, +) -> PointFiveUploadClient: + return PointFiveUploadClient( + api_key="p5tu_testkey", + api_url=api_url, + http_client=http_client, + max_retries=max_retries, + sleep=_no_backoff, + validate_upload_url=validate_upload_url, + ) + + +def _presigned_for(upload_url: str) -> httpx.Response: + return _response(200, {"uploadUrl": upload_url, "objectKey": OBJECT_KEY, "expiresAt": "2026-08-25T14:35:00Z"}) + + +@pytest.mark.asyncio +async def test_uploads_the_body_to_the_url_the_api_returned(): + http_client = FakeHTTPClient() + + outcome = await _client(http_client).upload(BODY) + + assert outcome == OBJECT_KEY + assert http_client.put_calls[0]["url"] == UPLOAD_URL + assert http_client.put_calls[0]["data"] == BODY + + +@pytest.mark.asyncio +async def test_presign_request_is_authenticated_and_sized(): + http_client = FakeHTTPClient() + + await _client(http_client).upload(BODY) + + call = http_client.presign_calls[0] + assert call["url"] == "https://api.pointfive.co/api/v1/ingestion/upload-url" + assert call["headers"]["Authorization"] == "Bearer p5tu_testkey" + assert call["json"] == {"kind": "LITELLM", "byteCount": len(BODY)} + + +@pytest.mark.asyncio +async def test_a_trailing_slash_on_the_api_url_is_tolerated(): + """A pasted URL often ends in a slash; it must not produce a double slash in the path.""" + http_client = FakeHTTPClient() + + await _client(http_client, api_url=API_URL + "/").upload(BODY) + + assert http_client.presign_calls[0]["url"] == "https://api.pointfive.co/api/v1/ingestion/upload-url" + + +@pytest.mark.asyncio +async def test_no_bearer_token_is_sent_to_the_presigned_url(): + """The URL carries its own authorization, so the api key must not travel with it.""" + http_client = FakeHTTPClient() + + await _client(http_client).upload(BODY) + + assert "Authorization" not in http_client.put_calls[0]["headers"] + + +@pytest.mark.asyncio +async def test_the_upload_pins_the_host_and_never_follows_a_redirect(): + http_client = FakeHTTPClient() + + await _client(http_client).upload(BODY) + + call = http_client.put_calls[0] + assert call["headers"]["Host"] == "uploads.example.invalid" + assert call["follow_redirects"] is False + + +@pytest.mark.asyncio +async def test_a_redirected_upload_is_refused_rather_than_followed(): + """A presigned URL never redirects legitimately; following one is how a bad endpoint reaches inside.""" + http_client = FakeHTTPClient(put=[httpx.Response(301, headers={"location": "http://169.254.169.254/"})]) + + outcome = await _client(http_client).upload(BODY) + + assert outcome == PointFiveUploadFailure( + "presigned upload redirected with 301, refusing to follow", retryable=False + ) + assert len(http_client.put_calls) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "upload_url", + [ + "http://169.254.169.254/latest/meta-data", + "https://10.0.0.7/internal/bucket/object", + "http://127.0.0.1:9000/bucket/object", + ], +) +async def test_an_upload_url_inside_the_network_is_refused_before_any_bytes_leave(upload_url): + http_client = FakeHTTPClient(presign=[_presigned_for(upload_url)]) + + outcome = await _client(http_client, validate_upload_url=validate_url).upload(BODY) + + assert isinstance(outcome, PointFiveUploadFailure) + assert not outcome.retryable + assert outcome.detail.startswith("presigned upload url refused: ") + assert http_client.put_calls == [] + + +@pytest.mark.asyncio +async def test_an_operator_can_switch_destination_validation_off(monkeypatch): + """litellm.user_url_validation is the proxy-wide switch every SSRF guard honours.""" + monkeypatch.setattr(litellm, "user_url_validation", False) + http_client = FakeHTTPClient(presign=[_presigned_for("http://10.0.0.7/bucket/object")]) + + outcome = await _client(http_client, validate_upload_url=validate_url).upload(BODY) + + assert outcome == OBJECT_KEY + assert http_client.put_calls[0]["url"] == "http://10.0.0.7/bucket/object" + assert "Host" not in http_client.put_calls[0]["headers"] + + +@pytest.mark.asyncio +async def test_the_object_is_declared_as_gzipped_ndjson(): + http_client = FakeHTTPClient() + + await _client(http_client).upload(BODY) + + assert http_client.put_calls[0]["headers"]["Content-Encoding"] == "gzip" + assert http_client.put_calls[0]["headers"]["Content-Type"] == "application/x-ndjson" + + +@pytest.mark.asyncio +async def test_each_retry_presigns_again(): + """A retry must never reuse a URL that was consumed or has expired.""" + http_client = FakeHTTPClient(put=[httpx.Response(503), _accepted()]) + + outcome = await _client(http_client).upload(BODY) + + assert outcome == OBJECT_KEY + assert len(http_client.presign_calls) == 2 + assert len(http_client.put_calls) == 2 + + +@pytest.mark.asyncio +async def test_retryable_upload_failure_gives_up_after_max_retries(): + http_client = FakeHTTPClient(put=[httpx.Response(503)]) + + outcome = await _client(http_client, max_retries=2).upload(BODY) + + assert outcome == PointFiveUploadFailure("presigned upload returned 503, gave up after 2 attempts", retryable=True) + assert len(http_client.put_calls) == 2 + + +@pytest.mark.asyncio +async def test_rejected_upload_is_not_retried(): + http_client = FakeHTTPClient(put=[httpx.Response(403)]) + + outcome = await _client(http_client).upload(BODY) + + assert outcome == PointFiveUploadFailure("presigned upload returned 403", retryable=False) + assert len(http_client.put_calls) == 1 + + +@pytest.mark.asyncio +async def test_bad_api_key_is_not_retried(): + http_client = FakeHTTPClient(presign=[httpx.Response(401)]) + + outcome = await _client(http_client).upload(BODY) + + assert outcome == PointFiveUploadFailure("pointfive api returned 401", retryable=False) + assert http_client.put_calls == [] + + +@pytest.mark.asyncio +async def test_the_reason_for_a_refusal_is_surfaced(): + """A 403 means the key no longer maps to an integration; the operator needs to read why.""" + http_client = FakeHTTPClient(presign=[_refused(403, "no integration accepts uploads from this api key")]) + + outcome = await _client(http_client).upload(BODY) + + assert outcome == PointFiveUploadFailure( + "pointfive api returned 403, no integration accepts uploads from this api key", retryable=False + ) + assert http_client.put_calls == [] + + +@pytest.mark.asyncio +async def test_api_server_error_is_retried(): + http_client = FakeHTTPClient(presign=[httpx.Response(503), _presigned()]) + + outcome = await _client(http_client).upload(BODY) + + assert outcome == OBJECT_KEY + assert len(http_client.presign_calls) == 2 + + +@pytest.mark.asyncio +async def test_too_many_requests_is_retried(): + http_client = FakeHTTPClient(presign=[httpx.Response(429), _presigned()]) + + outcome = await _client(http_client).upload(BODY) + + assert outcome == OBJECT_KEY + assert len(http_client.presign_calls) == 2 + + +@pytest.mark.asyncio +async def test_unreachable_api_is_retried_then_reported_as_retryable(): + http_client = FakeHTTPClient(presign=(ConnectionError("down"),)) + + outcome = await _client(http_client, max_retries=2).upload(BODY) + + assert isinstance(outcome, PointFiveUploadFailure) + assert outcome.retryable + assert "unreachable" in outcome.detail + assert len(http_client.presign_calls) == 2 + + +@pytest.mark.asyncio +async def test_malformed_api_body_is_not_retried(): + http_client = FakeHTTPClient(presign=[_response(200, {"objectKey": "k"})]) + + outcome = await _client(http_client).upload(BODY) + + assert outcome == PointFiveUploadFailure("pointfive api returned an unreadable body", retryable=False) + assert http_client.put_calls == [] + + +@pytest.mark.asyncio +async def test_a_body_that_is_not_json_is_reported_as_unreadable(): + http_client = FakeHTTPClient(presign=(httpx.Response(200, text="gateway"),)) + + outcome = await _client(http_client).upload(BODY) + + assert outcome == PointFiveUploadFailure("pointfive api returned an unreadable body", retryable=False) + assert http_client.put_calls == [] + + +@pytest.mark.asyncio +async def test_ping_reports_a_live_shipper(): + http_client = FakeHTTPClient(presign=(_no_content(),)) + + failure = await _client(http_client).ping() + + assert failure is None + assert http_client.presign_calls[0]["url"] == "https://api.pointfive.co/api/v1/ingestion/ping" + assert http_client.presign_calls[0]["json"] == {"kind": "LITELLM"} + + +@pytest.mark.asyncio +async def test_ping_surfaces_a_revoked_key(): + http_client = FakeHTTPClient(presign=(_refused(403, "no integration accepts uploads from this api key"),)) + + failure = await _client(http_client).ping() + + assert failure is not None + assert not failure.retryable + assert "no integration accepts uploads from this api key" in failure.detail + + +@pytest.mark.asyncio +async def test_ping_surfaces_an_unreachable_api(): + http_client = FakeHTTPClient(presign=(ConnectionError("down"),)) + + failure = await _client(http_client).ping() + + assert failure is not None + assert failure.retryable + + +@pytest.mark.asyncio +async def test_a_transport_fault_on_the_upload_itself_is_retryable(): + http_client = FakeHTTPClient(put=(ConnectionError("reset"),)) + + outcome = await _client(http_client, max_retries=1).upload(BODY) + + assert isinstance(outcome, PointFiveUploadFailure) + assert outcome.retryable + assert "presigned upload unreachable" in outcome.detail + + +@pytest.mark.asyncio +async def test_a_client_that_may_not_try_at_all_says_so(): + """max_upload_retries is validated as >= 1, so this guards the loop against a future zero.""" + outcome = await _client(FakeHTTPClient(), max_retries=0).upload(BODY) + + assert outcome == PointFiveUploadFailure("max_upload_retries must be at least 1", retryable=False) From 6b27a16e67e031325e8aeec8fe76b4a91885bfc1 Mon Sep 17 00:00:00 2001 From: Yinon Kahta Date: Thu, 27 Aug 2026 13:05:58 +0300 Subject: [PATCH 304/411] feat(pointfive): add batching logger callback --- litellm/__init__.py | 4 + litellm/integrations/pointfive/__init__.py | 5 + litellm/integrations/pointfive/logger.py | 304 +++++++ litellm/litellm_core_utils/redact_messages.py | 13 + .../integrations/pointfive/test_logger.py | 759 ++++++++++++++++++ 5 files changed, 1085 insertions(+) create mode 100644 litellm/integrations/pointfive/__init__.py create mode 100644 litellm/integrations/pointfive/logger.py create mode 100644 tests/test_litellm/integrations/pointfive/test_logger.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 2a85a3d4200..157aed12a83 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -45,9 +45,11 @@ from typing import ( TYPE_CHECKING, Union, ) +from collections.abc import Mapping from litellm.types.integrations.datadog import DatadogInitParams from litellm.types.integrations.newrelic import NewRelicInitParams from litellm.litellm_core_utils.core_helpers import drop_params_env_flag +from litellm.types.integrations.pointfive import PointFiveInitParams from litellm._logging import ( set_verbose, _turn_on_debug, @@ -154,6 +156,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "smtp_email", "deepeval", "s3_v2", + "pointfive", "aws_sqs", "vector_store_pre_call_hook", "dotprompt", @@ -439,6 +442,7 @@ s3_audit_callback_params: Optional[Dict] = None datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None datadog_params: Optional[Union[DatadogInitParams, Dict]] = None newrelic_params: Optional[Union[NewRelicInitParams, Dict]] = None +pointfive_params: Optional[Union[PointFiveInitParams, Mapping[str, object]]] = None aws_sqs_callback_params: Optional[Dict] = None generic_logger_headers: Optional[Dict] = None default_key_generate_params: Optional[Dict] = None diff --git a/litellm/integrations/pointfive/__init__.py b/litellm/integrations/pointfive/__init__.py new file mode 100644 index 00000000000..1f3ca3c65c7 --- /dev/null +++ b/litellm/integrations/pointfive/__init__.py @@ -0,0 +1,5 @@ +"""PointFive logging integration for LiteLLM.""" + +from litellm.integrations.pointfive.logger import PointFiveLogger + +__all__ = ("PointFiveLogger",) diff --git a/litellm/integrations/pointfive/logger.py b/litellm/integrations/pointfive/logger.py new file mode 100644 index 00000000000..c352dac11e7 --- /dev/null +++ b/litellm/integrations/pointfive/logger.py @@ -0,0 +1,304 @@ +""" +PointFive logging integration. + +Buffers ``StandardLoggingPayload`` records and ships each flush as one gzipped +newline-delimited JSON object, rather than one object per request. Uploads go through a +presigned URL issued by the PointFive API, so the proxy needs no cloud credentials and +runs unchanged wherever it is hosted. +""" + +import asyncio +from collections.abc import Mapping +from datetime import datetime +from typing import Final + +import litellm +from litellm._logging import verbose_logger +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.integrations.pointfive.payload import chunk_lines, encode_lines, serialize_records +from litellm.integrations.pointfive.upload_client import PointFiveUploadClient, PointFiveUploadError +from litellm.litellm_core_utils.redact_messages import ( + redacted_standard_logging_payload, + should_redact_message_logging, +) +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, httpxSpecialProvider +from litellm.secret_managers.main import get_secret_str +from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus +from litellm.types.integrations.pointfive import DEFAULT_API_URL, PointFiveInitParams, PointFiveUploadFailure + +_ENV_REFERENCE_PREFIX: Final = "os.environ/" + + +def _resolved_secret(value: str | None) -> str | None: + """ + Resolve a config value that may name a secret, in any shape the secret manager accepts. + + A reference that resolves to nothing stays unresolved rather than falling back to its own + text, so an unset ``os.environ/NAME`` reports a missing key instead of being sent as one. + """ + if value is None: + return None + resolved: Final = get_secret_str(value) + if resolved: + return resolved + return None if value.startswith(_ENV_REFERENCE_PREFIX) else value + + +def _configured_params() -> PointFiveInitParams: + """Read ``litellm.pointfive_params``, validating a raw config dict on the way through.""" + configured: Final = litellm.pointfive_params + if isinstance(configured, PointFiveInitParams): + return configured + if isinstance(configured, Mapping): + return PointFiveInitParams.model_validate(configured) + return PointFiveInitParams() + + +def _resolved_api_key(params: PointFiveInitParams) -> str | None: + """Prefer the configured key, falling back to the environment the proxy UI writes.""" + return _resolved_secret(params.api_key) or get_secret_str("POINTFIVE_API_KEY") + + +def _resolved_api_url(params: PointFiveInitParams) -> str: + """Prefer the configured url, then the environment, then the public endpoint.""" + return _resolved_secret(params.api_url) or get_secret_str("POINTFIVE_API_URL") or DEFAULT_API_URL + + +def _upload_client_for(params: PointFiveInitParams) -> PointFiveUploadClient: + """ + Build an upload client for the key and url configured right now. + + Resolved per call rather than kept: the proxy ui writes new values into the + environment of a running proxy, and reading them once would need a restart to take + effect. ``get_async_httpx_client`` is cached, so this reuses the same connections. + """ + api_key: Final = _resolved_api_key(params) + if not api_key: + raise ValueError( + "pointfive logging requires an api key. Set POINTFIVE_API_KEY, or " + "litellm_settings.pointfive_params.api_key in config.yaml" + ) + return PointFiveUploadClient( + api_key=api_key, + api_url=_resolved_api_url(params), + http_client=get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback), + max_retries=params.max_upload_retries, + ) + + +class PointFiveLogger(CustomBatchLogger): + """Batching callback that ships LiteLLM request logs to PointFive.""" + + preserve_events_added_during_flush = True + + def __init__( + self, + params: PointFiveInitParams | None = None, + upload_client: PointFiveUploadClient | None = None, + start_periodic_flush: bool = True, + ) -> None: + resolved: Final = params if params is not None else _configured_params() + self.max_batch_bytes: Final = resolved.max_batch_bytes + self.params: Final = resolved + self.given_upload_client: Final = upload_client + if upload_client is None: + _upload_client_for(resolved) # refuse to start without a key, rather than at the first flush + super().__init__( + flush_lock=asyncio.Lock(), + batch_size=resolved.batch_size, + flush_interval=resolved.flush_interval, + turn_off_message_logging=bool(resolved.turn_off_message_logging), + ) + self._flushing: bool = False + self._batch_flush_task: asyncio.Task[None] | None = None + self._periodic_flush_task: asyncio.Task[None] | None = ( + self._start_periodic_flush_task() if start_periodic_flush else None + ) + + @property + def upload_client(self) -> PointFiveUploadClient: + """The client for the currently configured key and url, so a ui edit needs no restart.""" + if self.given_upload_client is not None: + return self.given_upload_client + return _upload_client_for(self.params) + + def _start_periodic_flush_task(self) -> asyncio.Task[None] | None: + """Start the periodic flush only once an event loop is actually running.""" + try: + loop: Final = asyncio.get_running_loop() + except RuntimeError: + return None + return loop.create_task(self.periodic_flush()) + + def _start_batch_flush_task(self) -> None: + """ + Upload a full batch in the background, so no request waits on PointFive. + + Awaiting it here put the upload, its retries and their backoff on the caller's + path, and a hung api held a response open for as long as the attempts took. + """ + if self._batch_flush_task is not None and not self._batch_flush_task.done(): + return + try: + loop: Final = asyncio.get_running_loop() + except RuntimeError: + return + self._batch_flush_task = loop.create_task(self.flush_queue(skip_if_flushing=True)) + + def _flush_task_is_alive(self) -> bool: + """A task whose loop has been closed never runs again, yet never reports itself done.""" + task: Final = self._periodic_flush_task + return task is not None and not task.done() and not task.get_loop().is_closed() + + async def periodic_flush(self) -> None: + """ + Report in straight away, then flush on the interval as usual. + + The inherited loop sleeps first, so a proxy that has just loaded the callback says + nothing for a whole interval, five minutes by default. PointFive shows the integration + as still waiting for its first call for all that time, which reads as a broken setup + rather than an idle one. An empty queue makes this first cycle a ping, so a proxy with + no traffic yet announces itself without uploading an object that holds no records. + """ + await self.flush_queue(skip_if_flushing=True) + await super().periodic_flush() + + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime, + end_time: datetime, + ) -> None: + await self._enqueue(kwargs) + + async def async_log_failure_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime, + end_time: datetime, + ) -> None: + await self._enqueue(kwargs) + + async def _enqueue(self, kwargs: Mapping[str, object]) -> None: + """Buffer one record, flushing early once the batch threshold is reached.""" + try: + if not self._flush_task_is_alive(): + self._periodic_flush_task = self._start_periodic_flush_task() + + record: Final = self._record_for(kwargs) + if record is None: + verbose_logger.debug("pointfive: event carried no standard_logging_object, skipping") + return + + self.log_queue.append(record) + self._drop_overflow() + if len(self.log_queue) >= self.batch_size: + self._start_batch_flush_task() + except Exception: # noqa: BLE001 # logging must never break the request path + verbose_logger.exception("pointfive: failed to queue an event") + + def _record_for(self, kwargs: Mapping[str, object]) -> Mapping[str, object] | None: + """ + The record to buffer, redacted the way the framework would have redacted it. + + A success reaches a callback already redacted, an async failure does not, so both + the excluded-field list and this callback's own setting are applied here, then the + global, per-request and header settings that only the framework's predicate knows. + """ + details: Final = self.redact_standard_logging_payload_from_model_call_details( + dict(kwargs) # mutable-ok: both framework helpers take the call details as a dict + ) + payload: Final = details.get("standard_logging_object") + if not isinstance(payload, dict): + return None + if should_redact_message_logging(details): + return redacted_standard_logging_payload(payload) + return payload + + def _drop_overflow(self) -> None: + """ + Hold the queue to its cap as records arrive, not only after a flush has failed. + + Never while a flush is running: it holds a snapshot taken by length, and trimming + the front underneath it would make the post-flush drain remove records that arrived + during the upload and were never sent. The next arrival after the flush trims. + """ + if self._flushing: + return + overflow: Final = len(self.log_queue) - self.max_queue_size + if overflow <= 0: + return + del self.log_queue[:overflow] + verbose_logger.warning("pointfive: queue over %s records, dropped %s oldest", self.max_queue_size, overflow) + + async def flush_queue(self, skip_if_flushing: bool = False) -> None: + """ + Flush as usual, or report liveness when there is nothing to send. + + ``CustomBatchLogger`` skips an empty queue entirely, so without this an idle proxy + would look identical to a dead one. + + ``skip_if_flushing`` is what a full batch, and the loop's opening cycle, pass. Uploading one takes seconds, and + every event arriving meanwhile crosses the threshold too, so each would queue on the + flush lock and then ship the handful of records left behind it. That turns one burst + into a stream of tiny objects, which is what batching exists to avoid. The running + flush already carries what is queued, and the interval catches whatever it missed. + """ + if not self.log_queue: + await self._ping() + return + if skip_if_flushing and self._flushing: + return + + self._flushing = True + try: + await super().flush_queue() + finally: + self._flushing = False + + async def async_health_check(self) -> IntegrationHealthCheckStatus: + """Answer the proxy ui test button by asking the api whether it accepts this key.""" + try: + failure: Final = await self.upload_client.ping() + except ValueError as missing_key: + return IntegrationHealthCheckStatus(status="unhealthy", error_message=str(missing_key)) + if failure is not None: + return IntegrationHealthCheckStatus(status="unhealthy", error_message=failure.detail) + return IntegrationHealthCheckStatus(status="healthy", error_message=None) + + async def _ping(self) -> None: + """Report liveness, never failing the flush over it.""" + try: + failure: Final = await self.upload_client.ping() + except ValueError as missing_key: + verbose_logger.warning("pointfive: liveness ping skipped, %s", missing_key) + return + if failure is not None: + verbose_logger.warning("pointfive: liveness ping failed, %s", failure.detail) + + async def async_send_batch(self) -> None: + """ + Upload everything queued, split into objects of at most ``max_batch_bytes``. + + A retryable failure propagates so ``CustomBatchLogger`` keeps the rest of the batch + for the next flush; the records already shipped or already refused leave the queue + first, so a retry re-sends at most the object that failed. A rejection the server + will refuse again drops that object, since holding it would block every record + queued behind it. + """ + pending: Final = tuple(self.log_queue) + if not pending: + return + + client: Final = self.upload_client + chunks: Final = chunk_lines(serialize_records(pending), self.max_batch_bytes) + for index, chunk in enumerate(chunks): + outcome = await client.upload(await encode_lines(chunk)) + if not isinstance(outcome, PointFiveUploadFailure): + continue + if outcome.retryable: + del self.log_queue[: sum(len(shipped) for shipped in chunks[:index])] + raise PointFiveUploadError(outcome.detail) + verbose_logger.error("pointfive: dropping %s records, %s", len(chunk), outcome.detail) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 9402d465712..a567854d767 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -162,6 +162,19 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str): output_item["arguments"] = redacted_str +def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[str, object]: + """ + Return a copy of a ``StandardLoggingPayload`` with its messages and response redacted. + + The success path redacts through ``perform_redaction`` before a callback ever sees the + payload, but the failure path does not, so a callback that batches both has to redact + the ones it is handed. + """ + redacted: Final = copy.deepcopy(dict(payload)) # mutable-ok: redacted in place below + _redact_standard_logging_object({"standard_logging_object": redacted}) # mutable-ok: the callee's shape + return redacted + + def _redact_standard_logging_object(model_call_details: dict): """Redact messages and response inside standard_logging_object if present.""" standard_logging_object: Final = model_call_details.get("standard_logging_object") diff --git a/tests/test_litellm/integrations/pointfive/test_logger.py b/tests/test_litellm/integrations/pointfive/test_logger.py new file mode 100644 index 00000000000..52570751c74 --- /dev/null +++ b/tests/test_litellm/integrations/pointfive/test_logger.py @@ -0,0 +1,759 @@ +import asyncio +import gzip +import json +import logging +from collections.abc import Callable + +import pytest + +from litellm.integrations.pointfive.logger import PointFiveLogger +from litellm.integrations.pointfive.upload_client import PointFiveUploadError +from litellm.types.integrations.pointfive import DEFAULT_API_URL, PointFiveInitParams, PointFiveUploadFailure + +OBJECT_KEY = "some/object.ndjson.gz" + + +class FakeUploadClient: + """Records the objects a flush produced, so tests can read what would have shipped.""" + + def __init__( + self, + outcomes: list[str | PointFiveUploadFailure] | None = None, + ping_failure: PointFiveUploadFailure | None = None, + ) -> None: + self.outcomes = outcomes or [OBJECT_KEY] + self.bodies: list[bytes] = [] + self.on_upload: Callable[[], None] | None = None + self.ping_failure = ping_failure + self.pings = 0 + + async def ping(self) -> PointFiveUploadFailure | None: + self.pings += 1 + return self.ping_failure + + async def upload(self, body: bytes) -> str | PointFiveUploadFailure: + if self.on_upload is not None: + self.on_upload() + self.bodies.append(body) + return self.outcomes.pop(0) if len(self.outcomes) > 1 else self.outcomes[0] + + def records(self) -> list[dict]: + return [json.loads(line) for body in self.bodies for line in gzip.decompress(body).decode().splitlines()] + + +def _logger(upload_client: FakeUploadClient, **params) -> PointFiveLogger: + return PointFiveLogger(params=PointFiveInitParams(**params), upload_client=upload_client) + + +def _event(request_id: str, size: int = 0) -> dict: + return {"standard_logging_object": {"id": request_id, "model": "gpt-4o", "blob": "x" * size}} + + +@pytest.mark.asyncio +async def test_a_flush_ships_one_object_holding_every_buffered_record(): + """One object per flush is the whole point: s3_v2 sends one per request.""" + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=3) + + for request_id in ("a", "b", "c"): + await logger.async_log_success_event(_event(request_id), None, None, None) + + await _settle(logger) + assert len(upload_client.bodies) == 1 + assert [record["id"] for record in upload_client.records()] == ["a", "b", "c"] + + +@pytest.mark.asyncio +async def test_records_are_held_until_the_batch_is_full(): + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=3) + + await logger.async_log_success_event(_event("a"), None, None, None) + + assert upload_client.bodies == [] + assert len(logger.log_queue) == 1 + + +@pytest.mark.asyncio +async def test_failed_requests_are_logged_too(): + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=1) + + await logger.async_log_failure_event(_event("failed"), None, None, None) + + await _settle(logger) + assert [record["id"] for record in upload_client.records()] == ["failed"] + + +@pytest.mark.asyncio +async def test_an_event_without_a_standard_payload_is_skipped(): + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=1) + + await logger.async_log_success_event({"kwargs": "but no payload"}, None, None, None) + + assert upload_client.bodies == [] + assert logger.log_queue == [] + + +@pytest.mark.asyncio +async def test_a_batch_over_the_byte_cap_ships_as_several_objects(): + """Record count cannot bound an object: an unredacted payload dwarfs a redacted one.""" + cap = 600 + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=4, max_batch_bytes=cap) + + for request_id in ("a", "b", "c", "d"): + await logger.async_log_success_event(_event(request_id, size=200), None, None, None) + + await _settle(logger) + assert len(upload_client.bodies) > 1 + assert [record["id"] for record in upload_client.records()] == ["a", "b", "c", "d"] + assert all(len(gzip.decompress(body)) <= cap for body in upload_client.bodies) + + +@pytest.mark.asyncio +async def test_a_retryable_failure_keeps_the_batch_for_the_next_flush(): + upload_client = FakeUploadClient([PointFiveUploadFailure("upload target is down", retryable=True)]) + logger = _logger(upload_client, batch_size=2) + + for request_id in ("a", "b"): + await logger.async_log_success_event(_event(request_id), None, None, None) + + assert [record["id"] for record in logger.log_queue] == ["a", "b"] + + +@pytest.mark.asyncio +async def test_a_retryable_failure_surfaces_so_the_base_logger_can_preserve_it(): + upload_client = FakeUploadClient([PointFiveUploadFailure("upload target is down", retryable=True)]) + logger = _logger(upload_client, batch_size=99) + logger.log_queue.append(_event("a")["standard_logging_object"]) + + with pytest.raises(PointFiveUploadError, match="upload target is down"): + await logger.async_send_batch() + + +@pytest.mark.asyncio +async def test_a_rejected_batch_is_dropped_rather_than_blocking_the_queue(): + """Retrying a rejection forever would stall every record queued behind it.""" + upload_client = FakeUploadClient([PointFiveUploadFailure("object too large", retryable=False)]) + logger = _logger(upload_client, batch_size=2) + + for request_id in ("a", "b"): + await logger.async_log_success_event(_event(request_id), None, None, None) + + await _settle(logger) + assert logger.log_queue == [] + + +@pytest.mark.asyncio +async def test_records_already_queued_ship_with_the_event_that_triggers_the_flush(): + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=1) + logger.log_queue.append(_event("mid-flight")["standard_logging_object"]) + + await logger.async_log_success_event(_event("a"), None, None, None) + + await _settle(logger) + assert [record["id"] for record in upload_client.records()] == ["mid-flight", "a"] + assert logger.log_queue == [] + + +@pytest.mark.asyncio +async def test_a_record_that_arrives_mid_flush_is_kept_for_the_next_one(): + """The queue is drained by count, so a record appended mid-upload must survive.""" + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=1) + upload_client.on_upload = lambda: logger.log_queue.append(_event("late")["standard_logging_object"]) + + await logger.async_log_success_event(_event("first"), None, None, None) + + await _settle(logger) + assert [record["id"] for record in upload_client.records()] == ["first"] + assert [record["id"] for record in logger.log_queue] == ["late"] + + +def test_defaults_favour_fewer_larger_uploads_over_freshness(): + upload_client = FakeUploadClient() + + logger = _logger(upload_client) + + assert logger.batch_size == 1_000 + assert logger.flush_interval == 300 + assert logger.max_batch_bytes == 8 * 1024 * 1024 + + +def test_the_default_api_url_is_the_pointfive_ingress(monkeypatch): + """api.pointfive.co is the host the ingress serves; .com does not resolve to it.""" + monkeypatch.setenv("POINTFIVE_API_KEY", "p5tu_env") + + logger = PointFiveLogger() + + assert logger.upload_client.api_url == "https://api.pointfive.co/api/v1/ingestion" + + +def test_the_api_key_can_come_from_the_environment(monkeypatch): + """The proxy ui configures a callback by writing environment variables.""" + monkeypatch.setenv("POINTFIVE_API_KEY", "p5tu_from_env") + + logger = PointFiveLogger() + + assert logger.upload_client.api_key == "p5tu_from_env" + + +def test_the_api_url_can_come_from_the_environment(monkeypatch): + monkeypatch.setenv("POINTFIVE_API_KEY", "p5tu_env") + monkeypatch.setenv("POINTFIVE_API_URL", "https://api.staging.pointfive.co/api/v1/ingestion") + + logger = PointFiveLogger() + + assert logger.upload_client.api_url == "https://api.staging.pointfive.co/api/v1/ingestion" + + +def test_config_yaml_wins_over_the_environment(monkeypatch): + """A value set in config.yaml is explicit, so it outranks whatever the ui left behind.""" + monkeypatch.setenv("POINTFIVE_API_KEY", "p5tu_from_env") + monkeypatch.setenv("POINTFIVE_API_URL", "https://from-env.example/api/v1/ingestion") + + logger = PointFiveLogger( + params=PointFiveInitParams(api_key="p5tu_from_config", api_url="https://from-config.example/api/v1/ingestion") + ) + + assert logger.upload_client.api_key == "p5tu_from_config" + assert logger.upload_client.api_url == "https://from-config.example/api/v1/ingestion" + + +def test_a_missing_api_key_fails_at_startup_not_at_the_first_flush(monkeypatch): + monkeypatch.delenv("POINTFIVE_API_KEY", raising=False) + + with pytest.raises(ValueError, match="api key"): + PointFiveLogger(params=PointFiveInitParams()) + + +def test_an_api_key_can_be_an_environment_reference(monkeypatch): + """config.yaml spells secrets as `os.environ/NAME`, so the plugin must resolve one.""" + monkeypatch.setenv("POINTFIVE_TEST_KEY", "p5tu_from_env") + + logger = PointFiveLogger(params=PointFiveInitParams(api_key="os.environ/POINTFIVE_TEST_KEY")) + + assert logger.upload_client.api_key == "p5tu_from_env" + + +def test_params_are_read_from_litellm_settings(monkeypatch): + import litellm + + monkeypatch.setattr(litellm, "pointfive_params", {"api_key": "p5tu_configured", "batch_size": 7}) + + logger = PointFiveLogger() + + assert logger.upload_client.api_key == "p5tu_configured" + assert logger.batch_size == 7 + + +def test_an_out_of_range_setting_is_rejected(): + with pytest.raises(ValueError, match="batch_size"): + PointFiveInitParams(api_key="p5tu_k", batch_size=0) + + +@pytest.mark.asyncio +async def test_an_idle_flush_reports_liveness_instead_of_uploading(): + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=99) + + await logger.flush_queue() + + assert upload_client.pings == 1 + assert upload_client.bodies == [] + + +@pytest.mark.asyncio +async def test_a_flush_with_records_uploads_and_does_not_ping(): + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=1) + + await logger.async_log_success_event(_event("a"), None, None, None) + + await _settle(logger) + assert upload_client.pings == 0 + assert len(upload_client.bodies) == 1 + + +@pytest.mark.asyncio +async def test_a_failed_ping_does_not_raise(): + """Liveness is bookkeeping; a proxy must not see errors from it.""" + upload_client = FakeUploadClient(ping_failure=PointFiveUploadFailure("api down", retryable=True)) + logger = _logger(upload_client, batch_size=99) + + await logger.flush_queue() + + assert upload_client.pings == 1 + + +@pytest.mark.asyncio +async def test_health_check_is_healthy_when_the_api_accepts_the_key(): + upload_client = FakeUploadClient() + + assert await _logger(upload_client).async_health_check() == {"status": "healthy", "error_message": None} + assert upload_client.pings == 1 + + +@pytest.mark.asyncio +async def test_health_check_reports_why_the_api_refused(): + """The ui test button shows this message, so a rejected key has to say so rather than pass.""" + upload_client = FakeUploadClient(ping_failure=PointFiveUploadFailure("key was revoked", retryable=False)) + + outcome = await _logger(upload_client).async_health_check() + + assert outcome == {"status": "unhealthy", "error_message": "key was revoked"} + + +def test_the_client_follows_a_key_and_url_changed_after_startup(monkeypatch): + """ + The proxy ui writes new values into a running proxy's environment. + + Reading them once at construction would leave the logger talking to the old endpoint + until someone restarted the proxy. + """ + monkeypatch.setenv("POINTFIVE_API_KEY", "p5tu_first") + monkeypatch.setenv("POINTFIVE_API_URL", "https://first.example.invalid/api/v1/ingestion") + logger = PointFiveLogger(params=PointFiveInitParams()) + + assert logger.upload_client.api_key == "p5tu_first" + assert logger.upload_client.api_url == "https://first.example.invalid/api/v1/ingestion" + + monkeypatch.setenv("POINTFIVE_API_KEY", "p5tu_second") + monkeypatch.setenv("POINTFIVE_API_URL", "https://second.example.invalid/api/v1/ingestion") + + assert logger.upload_client.api_key == "p5tu_second" + assert logger.upload_client.api_url == "https://second.example.invalid/api/v1/ingestion" + + +def test_a_configured_key_still_wins_over_the_environment(monkeypatch): + monkeypatch.setenv("POINTFIVE_API_KEY", "p5tu_from_env") + logger = PointFiveLogger(params=PointFiveInitParams(api_key="p5tu_from_config")) + + assert logger.upload_client.api_key == "p5tu_from_config" + + +@pytest.mark.asyncio +async def test_health_check_says_so_when_the_key_was_removed(monkeypatch): + monkeypatch.setenv("POINTFIVE_API_KEY", "p5tu_present") + logger = PointFiveLogger(params=PointFiveInitParams()) + monkeypatch.delenv("POINTFIVE_API_KEY") + + outcome = await logger.async_health_check() + + assert outcome["status"] == "unhealthy" + assert "requires an api key" in (outcome["error_message"] or "") + + +def _pending_flush_tasks() -> tuple[asyncio.Task, ...]: + return tuple(task for task in asyncio.all_tasks() if "periodic_flush" in str(task.get_coro())) + + +@pytest.mark.asyncio +async def test_a_one_shot_logger_leaves_no_flush_task_behind(): + """ + A health check builds a logger for a single answer and drops it. + + Without this, every check would leave a flusher running that keeps pinging for the + lifetime of the proxy. + """ + before = _pending_flush_tasks() + + logger = PointFiveLogger(params=PointFiveInitParams(), upload_client=FakeUploadClient(), start_periodic_flush=False) + + assert logger._periodic_flush_task is None + assert _pending_flush_tasks() == before + + +@pytest.mark.asyncio +async def test_the_logger_flushes_periodically_by_default(): + logger = PointFiveLogger(params=PointFiveInitParams(), upload_client=FakeUploadClient()) + + assert logger._periodic_flush_task is not None + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_params_already_built_are_used_as_they_are(monkeypatch): + """config.yaml is validated once into a params object; a second validation would be wasted.""" + import litellm + + monkeypatch.setattr(litellm, "pointfive_params", PointFiveInitParams(max_batch_bytes=4096)) + + logger = PointFiveLogger(upload_client=FakeUploadClient()) + + assert logger.max_batch_bytes == 4096 + + +@pytest.mark.asyncio +async def test_a_dead_flush_task_is_restarted_by_the_next_event(): + """A cancelled or crashed flusher would otherwise leave the queue growing forever.""" + logger = _logger(FakeUploadClient()) + logger._periodic_flush_task.cancel() + await asyncio.sleep(0) # let the cancellation land, so the task reports itself done + + await logger.async_log_success_event(_event("after-cancel"), None, None, None) + + assert logger._periodic_flush_task is not None + assert not logger._periodic_flush_task.done() + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_a_failure_while_queueing_never_breaks_the_request(): + """Logging sits on the request path, so a fault here must not surface to the caller.""" + + class ExplodingQueue(list): + def append(self, _item): + raise RuntimeError("queue is broken") + + upload_client = FakeUploadClient() + logger = _logger(upload_client) + logger.log_queue = ExplodingQueue() + + await logger.async_log_success_event(_event("boom"), None, None, None) + + logger.log_queue = [] + await logger.async_log_success_event(_event("after-the-fault"), None, None, None) + assert [record["id"] for record in logger.log_queue] == ["after-the-fault"] + + +@pytest.mark.asyncio +async def test_a_flush_with_nothing_queued_uploads_nothing(): + upload_client = FakeUploadClient() + logger = _logger(upload_client) + + await logger.async_send_batch() + + assert upload_client.bodies == [] + + +@pytest.mark.asyncio +async def test_the_idle_ping_is_skipped_when_the_key_was_removed(monkeypatch, caplog): + """A key pulled mid-flight must not turn the periodic flush into an exception.""" + monkeypatch.setenv("POINTFIVE_API_KEY", "p5tu_present") + logger = PointFiveLogger(params=PointFiveInitParams()) + monkeypatch.delenv("POINTFIVE_API_KEY") + + with caplog.at_level(logging.WARNING): + await logger.flush_queue() + + assert "liveness ping skipped" in caplog.text + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_a_full_batch_stands_down_while_a_flush_is_already_running(): + """ + Under load every event landing mid-upload also crosses the batch threshold. + + Letting each one flush turns a single burst into a stream of tiny objects, which is + what batching exists to avoid, so a full batch defers to the flush already running. + """ + upload_client = FakeUploadClient() + # No periodic task: this test drives the flushes itself, and the loop's opening cycle + # would otherwise ship the queue it seeds below. + logger = PointFiveLogger( + params=PointFiveInitParams(batch_size=2), + upload_client=upload_client, + start_periodic_flush=False, + ) + release = asyncio.Event() + finish_upload = upload_client.upload + + async def held_upload(body: bytes): + await release.wait() + return await finish_upload(body) + + upload_client.upload = held_upload + logger.log_queue.extend(_event(f"first-{index}")["standard_logging_object"] for index in range(2)) + + flushing = asyncio.create_task(logger.flush_queue()) + await asyncio.sleep(0) + + # Bounded: without the guard these block on the flush lock the held upload owns. + for index in range(6): + await asyncio.wait_for(logger.async_log_success_event(_event(f"mid-{index}"), None, None, None), timeout=2) + + assert upload_client.bodies == [] + + release.set() + await flushing + + assert len(upload_client.bodies) == 1 + assert [record["id"] for record in upload_client.records()] == ["first-0", "first-1"] + assert [record["id"] for record in logger.log_queue] == [f"mid-{index}" for index in range(6)] + + +async def _settle(logger) -> None: + """ + Wait out the flush a full batch schedules, the way the proxy's loop would. + + The upload runs off the request path now, and either the batch task or the periodic + loop can be the one carrying it, so this waits for whichever is in flight to finish. + """ + for _ in range(200): + await asyncio.sleep(0.001) + task = logger._batch_flush_task + if task is not None and not task.done(): + await task + if not logger._flushing: + return + raise AssertionError("the flush never finished") + + +async def _until(done: Callable[[], bool], ticks: int = 400) -> None: + """Wait for a condition the flush path reaches only after gzip finishes on a worker thread.""" + for _ in range(ticks): + if done(): + return + await asyncio.sleep(0.005) + raise AssertionError("condition never became true") + + +@pytest.mark.asyncio +async def test_a_new_logger_announces_itself_without_waiting_for_the_interval(): + """ + Configuring the callback must make the integration connect, with no traffic and no test click. + + The inherited loop sleeps a whole interval before its first flush, which left a freshly + configured proxy silent for five minutes and the integration looking unconfigured. + """ + upload_client = FakeUploadClient() + logger = _logger(upload_client) + + await asyncio.sleep(0) # let the flush task reach its first cycle + + assert upload_client.pings == 1 + assert upload_client.bodies == [] + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_the_first_cycle_ships_records_rather_than_announcing(): + """Announcing is only for an empty queue: records already waiting must go out as an upload.""" + upload_client = FakeUploadClient() + logger = PointFiveLogger( + params=PointFiveInitParams(batch_size=100), + upload_client=upload_client, + start_periodic_flush=False, + ) + logger.log_queue.append(_event("queued-before-start")["standard_logging_object"]) + + logger._periodic_flush_task = logger._start_periodic_flush_task() + await _until(lambda: bool(upload_client.bodies)) + + assert upload_client.pings == 0 + assert [record["id"] for record in upload_client.records()] == ["queued-before-start"] + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_a_failed_request_ships_redacted_when_message_logging_is_off(): + """ + Failure events skip the framework's redaction, so the callback has to redact what it buffers. + + Without this the prompt of every failed request reaches PointFive in full, even though + the integration was configured not to send message content. + """ + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=1, turn_off_message_logging=True) + event = _event("failed-request") + event["standard_logging_object"]["messages"] = [{"role": "user", "content": "my secret prompt"}] + event["standard_logging_object"]["response"] = "the secret answer" + + await logger.async_log_failure_event(event, None, None, None) + await _settle(logger) + + shipped = upload_client.records()[0] + assert "my secret prompt" not in json.dumps(shipped) + assert "the secret answer" not in json.dumps(shipped) + assert event["standard_logging_object"]["messages"][0]["content"] == "my secret prompt" + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_a_dead_loop_does_not_strand_the_flusher(): + """A task whose loop was closed never runs and never reports done, so it must be replaced.""" + logger = PointFiveLogger(params=PointFiveInitParams(), upload_client=FakeUploadClient(), start_periodic_flush=False) + stranded_loop = asyncio.new_event_loop() + forever = asyncio.sleep(3600) + logger._periodic_flush_task = stranded_loop.create_task(forever) + stranded_loop.close() + forever.close() + + await logger.async_log_success_event(_event("after-loop-close"), None, None, None) + + assert logger._periodic_flush_task is not None + assert logger._periodic_flush_task.get_loop() is asyncio.get_running_loop() + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_a_retry_does_not_resend_objects_that_already_landed(): + """ + A failure part way through a multi-object flush used to hand the whole batch back. + + Every record already shipped, and every record already refused for good, went out + again on the next flush, so PointFive received duplicates of both. + """ + upload_client = FakeUploadClient(outcomes=[OBJECT_KEY, PointFiveUploadFailure("service busy", retryable=True)]) + logger = PointFiveLogger( + params=PointFiveInitParams(max_batch_bytes=1), # one record per object + upload_client=upload_client, + start_periodic_flush=False, + ) + logger.log_queue.extend(_event(request_id)["standard_logging_object"] for request_id in ("first", "second")) + + with pytest.raises(PointFiveUploadError): + await logger.async_send_batch() + + assert [record["id"] for record in logger.log_queue] == ["second"] + + +@pytest.mark.asyncio +async def test_the_queue_stops_growing_at_its_cap_without_waiting_for_a_failure(): + """The base class trims only after a failed send, so a proxy that keeps flushing never trims.""" + logger = _logger(FakeUploadClient(), batch_size=10_000) + logger.max_queue_size = 3 + + for request_id in ("a", "b", "c", "d", "e"): + await logger.async_log_success_event(_event(request_id), None, None, None) + + assert [record["id"] for record in logger.log_queue] == ["c", "d", "e"] + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_a_failed_request_honours_the_global_redaction_setting(monkeypatch): + """ + Redaction can be turned on globally or per request, not only on this callback. + + The async failure path hands the payload over untouched, so a tenant could trigger a + provider failure and ship prompts that the operator had already asked to be redacted. + """ + import litellm + + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=1) + event = _event("globally-redacted") + event["standard_logging_object"]["messages"] = [{"role": "user", "content": "my secret prompt"}] + + await logger.async_log_failure_event(event, None, None, None) + + await _settle(logger) + assert "my secret prompt" not in json.dumps(upload_client.records()[0]) + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_excluded_fields_are_dropped_from_a_failed_request(monkeypatch): + """standard_logging_payload_excluded_fields drops a field entirely; failures skipped it too.""" + import litellm + + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["messages"]) + upload_client = FakeUploadClient() + logger = _logger(upload_client, batch_size=1, turn_off_message_logging=True) + event = _event("field-excluded") + event["standard_logging_object"]["messages"] = [{"role": "user", "content": "my secret prompt"}] + + await logger.async_log_failure_event(event, None, None, None) + await _settle(logger) + + shipped = upload_client.records()[0] + assert "messages" not in shipped + assert shipped["id"] == "field-excluded" + logger._periodic_flush_task.cancel() + + +def _held_upload(upload_client: FakeUploadClient, release: asyncio.Event) -> None: + finish = upload_client.upload + + async def held(body: bytes): + await release.wait() + return await finish(body) + + upload_client.upload = held + + +@pytest.mark.asyncio +async def test_a_full_batch_does_not_hold_the_request(): + """ + The upload belongs off the request path. + + Awaiting it inline meant a hung PointFive api held the caller's response open for as + long as the attempts and their backoff took. + """ + upload_client = FakeUploadClient() + release = asyncio.Event() + _held_upload(upload_client, release) + logger = _logger(upload_client, batch_size=1) + + await asyncio.wait_for(logger.async_log_success_event(_event("first"), None, None, None), timeout=2) + + assert upload_client.bodies == [] + release.set() + await _settle(logger) + assert [record["id"] for record in upload_client.records()] == ["first"] + logger._periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_records_arriving_during_a_flush_survive_the_queue_cap(): + """ + The flush drains by count, so trimming the front underneath it loses records. + + Records that arrived while the upload was in flight would be deleted by that drain + without ever being sent. + """ + upload_client = FakeUploadClient() + release = asyncio.Event() + _held_upload(upload_client, release) + logger = PointFiveLogger( + params=PointFiveInitParams(batch_size=2), + upload_client=upload_client, + start_periodic_flush=False, + ) + logger.max_queue_size = 2 + logger.log_queue.extend(_event(request_id)["standard_logging_object"] for request_id in ("a", "b")) + + flushing = asyncio.create_task(logger.flush_queue()) + await asyncio.sleep(0.01) + for request_id in ("c", "d", "e"): + await logger.async_log_success_event(_event(request_id), None, None, None) + release.set() + await flushing + + assert [record["id"] for record in upload_client.records()] == ["a", "b"] + assert [record["id"] for record in logger.log_queue] == ["c", "d", "e"] + logger._periodic_flush_task.cancel() + + +def test_an_unset_env_reference_is_never_used_as_the_key(monkeypatch): + """ + A config that names a missing variable has no key, and must say so. + + Falling back to the reference text sent the literal "os.environ/NAME" as the bearer + token, so the callback started and every upload was rejected for the wrong reason. + """ + monkeypatch.delenv("POINTFIVE_API_KEY", raising=False) + monkeypatch.delenv("POINTFIVE_MISSING_KEY", raising=False) + + with pytest.raises(ValueError, match="requires an api key"): + PointFiveLogger( + params=PointFiveInitParams(api_key="os.environ/POINTFIVE_MISSING_KEY"), + start_periodic_flush=False, + ) + + +def test_an_unset_url_reference_falls_back_to_the_public_endpoint(monkeypatch): + """An unresolved url reference must not become the destination the proxy uploads to.""" + from litellm.integrations.pointfive.logger import _resolved_api_url + + monkeypatch.delenv("POINTFIVE_API_URL", raising=False) + monkeypatch.delenv("POINTFIVE_MISSING_URL", raising=False) + + assert _resolved_api_url(PointFiveInitParams(api_url="os.environ/POINTFIVE_MISSING_URL")) == DEFAULT_API_URL From 224c7d62162f7ae5f47c67a8bafe1b90b7193dfe Mon Sep 17 00:00:00 2001 From: Yinon Kahta Date: Thu, 27 Aug 2026 13:05:58 +0300 Subject: [PATCH 305/411] feat(pointfive): register the pointfive callback --- .../litellm_core_utils/custom_logger_registry.py | 2 ++ litellm/litellm_core_utils/litellm_logging.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index 6449aa4d46e..6294f3bc577 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -43,6 +43,7 @@ from litellm.integrations.newrelic import NewRelicLogger from litellm.integrations.openmeter import OpenMeterLogger from litellm.integrations.opentelemetry import OpenTelemetry from litellm.integrations.opik.opik import OpikLogger +from litellm.integrations.pointfive import PointFiveLogger from litellm.integrations.posthog import PostHogLogger from litellm.integrations.prometheus import PrometheusLogger from litellm.integrations.s3_v2 import S3Logger @@ -95,6 +96,7 @@ class CustomLoggerRegistry: "agentops": AgentOps, "deepeval": DeepEvalLogger, "s3_v2": S3Logger, + "pointfive": PointFiveLogger, "aws_sqs": SQSLogger, "dynamic_rate_limiter": _PROXY_DynamicRateLimitHandler, "dynamic_rate_limiter_v3": _PROXY_DynamicRateLimitHandlerV3, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index de45414dbf3..cb9209be267 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -182,6 +182,7 @@ from ..integrations.lunary import LunaryLogger from ..integrations.newrelic import NewRelicLogger from ..integrations.openmeter import OpenMeterLogger from ..integrations.opik.opik import OpikLogger +from ..integrations.pointfive import PointFiveLogger from ..integrations.posthog import PostHogLogger from ..integrations.prompt_layer import PromptLayerLogger from ..integrations.s3 import S3Logger @@ -4376,6 +4377,14 @@ def _init_custom_logger_compatible_class( _s3_v2_logger: Final = S3V2Logger() _in_memory_loggers.append(_s3_v2_logger) return _s3_v2_logger + elif logging_integration == "pointfive": + for callback in _in_memory_loggers: + if isinstance(callback, PointFiveLogger): + return callback + + _pointfive_logger: Final = PointFiveLogger() + _in_memory_loggers.append(_pointfive_logger) + return _pointfive_logger elif logging_integration == "aws_sqs": for callback in _in_memory_loggers: if isinstance(callback, SQSLogger): @@ -5064,6 +5073,10 @@ def get_custom_logger_compatible_class( for callback in _in_memory_loggers: if isinstance(callback, S3V2Logger): return callback + elif logging_integration == "pointfive": + for callback in _in_memory_loggers: + if isinstance(callback, PointFiveLogger): + return callback elif logging_integration == "aws_sqs": for callback in _in_memory_loggers: if isinstance(callback, SQSLogger): From cb150846c276319d30ee976dd7bf1e0c550fd11c Mon Sep 17 00:00:00 2001 From: Yinon Kahta Date: Thu, 27 Aug 2026 13:05:58 +0300 Subject: [PATCH 306/411] feat(pointfive): list pointfive in the proxy callback registry --- litellm/proxy/_types.py | 9 +++++++++ .../proxy/test_pointfive_ui_callback.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/test_litellm/proxy/test_pointfive_ui_callback.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c22bb76629d..f496140e905 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3726,6 +3726,15 @@ class AllCallbacks(LiteLLMPydanticObjectBase): ], ) + pointfive: CallbackOnUI = CallbackOnUI( + litellm_callback_name="pointfive", + ui_callback_name="PointFive", + litellm_callback_params=[ # mutable-ok: the registry field is typed list + "POINTFIVE_API_KEY", + "POINTFIVE_API_URL", + ], + ) + class SpendLogsRouterMetadata(TypedDict): """ diff --git a/tests/test_litellm/proxy/test_pointfive_ui_callback.py b/tests/test_litellm/proxy/test_pointfive_ui_callback.py new file mode 100644 index 00000000000..13591d92005 --- /dev/null +++ b/tests/test_litellm/proxy/test_pointfive_ui_callback.py @@ -0,0 +1,15 @@ +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._types import AllCallbacks + + +def test_pointfive_is_offered_in_the_ui_callback_registry(): + """The proxy ui builds its form from this registry, so an absent entry is an absent form.""" + entry = AllCallbacks().pointfive + + assert entry.litellm_callback_name == "pointfive" + assert entry.ui_callback_name == "PointFive" + + +def test_the_ui_offers_the_two_settings_the_plugin_reads(): + """get_callback_env_vars is what the ui renders; it must match what the logger looks up.""" + assert tuple(CustomLogger.get_callback_env_vars("pointfive")) == ("POINTFIVE_API_KEY", "POINTFIVE_API_URL") From 29b02e921596f7dd177384e684d730d1f86d0d49 Mon Sep 17 00:00:00 2001 From: Yinon Kahta Date: Thu, 27 Aug 2026 13:05:58 +0300 Subject: [PATCH 307/411] feat(pointfive): add pointfive to the dashboard logging integrations --- litellm/integrations/callback_configs.json | 21 ++++++++ .../proxy/test_pointfive_dashboard_config.py | 47 ++++++++++++++++++ .../public/assets/logos/pointfive.png | Bin 0 -> 5284 bytes .../src/components/callback_info_helpers.tsx | 12 +++++ 4 files changed, 80 insertions(+) create mode 100644 tests/test_litellm/proxy/test_pointfive_dashboard_config.py create mode 100644 ui/litellm-dashboard/public/assets/logos/pointfive.png diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index c40b90cee25..85bfcc6e7ed 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -378,6 +378,27 @@ }, "description": "OpenTelemetry Logging Integration" }, + { + "id": "pointfive", + "displayName": "PointFive", + "logo": "pointfive.png", + "supports_key_team_logging": false, + "dynamic_params": { + "POINTFIVE_API_KEY": { + "type": "password", + "ui_name": "API Key", + "description": "PointFive API key, used to request an upload url for each batch of logs", + "required": true + }, + "POINTFIVE_API_URL": { + "type": "text", + "ui_name": "API URL", + "description": "PointFive API endpoint. Leave blank to use https://api.pointfive.co/api/v1/ingestion", + "required": false + } + }, + "description": "PointFive Logging Integration" + }, { "id": "s3", "displayName": "S3", diff --git a/tests/test_litellm/proxy/test_pointfive_dashboard_config.py b/tests/test_litellm/proxy/test_pointfive_dashboard_config.py new file mode 100644 index 00000000000..9e46e7f0b3e --- /dev/null +++ b/tests/test_litellm/proxy/test_pointfive_dashboard_config.py @@ -0,0 +1,47 @@ +import json +from pathlib import Path + +import litellm +from litellm.integrations.custom_logger import CustomLogger + + +def _dashboard_configs() -> tuple[dict, ...]: + path = Path(litellm.__file__).parent / "integrations" / "callback_configs.json" + return tuple(json.loads(path.read_text())) + + +def _pointfive_config() -> dict: + return next(config for config in _dashboard_configs() if config["id"] == "pointfive") + + +def test_pointfive_appears_in_the_dashboard_callback_dropdown(): + """The dropdown is served from callback_configs.json, so an entry only in the dashboard source is invisible.""" + entry = _pointfive_config() + + assert entry["displayName"] == "PointFive" + assert entry["supports_key_team_logging"] is False + assert entry["dynamic_params"]["POINTFIVE_API_KEY"]["required"] is True + assert entry["dynamic_params"]["POINTFIVE_API_KEY"]["type"] == "password" + assert entry["dynamic_params"]["POINTFIVE_API_URL"]["required"] is False + + +def test_the_dropdown_logo_asset_exists(): + """A logo the dashboard cannot resolve degrades silently to a letter tile.""" + logo = _pointfive_config()["logo"] + repo_root = Path(litellm.__file__).parent.parent + asset = repo_root / "ui" / "litellm-dashboard" / "public" / "assets" / "logos" / logo + + assert logo == "pointfive.png" + assert asset.is_file() + + +def test_the_dropdown_fields_are_the_env_vars_the_logger_reads(): + """ + The field names are the environment variables verbatim. + + The proxy would uppercase them either way, but naming them as stored means the edit + form finds the saved values and prefills them instead of showing blanks. + """ + fields = tuple(_pointfive_config()["dynamic_params"]) + + assert fields == tuple(CustomLogger.get_callback_env_vars("pointfive")) diff --git a/ui/litellm-dashboard/public/assets/logos/pointfive.png b/ui/litellm-dashboard/public/assets/logos/pointfive.png new file mode 100644 index 0000000000000000000000000000000000000000..4b7a6b8939ee49b967a8cd6cb89b3dd23f212cef GIT binary patch literal 5284 zcmcI|`9GBJ_y0YOF_Udn(@T;uLdaGk$u@&R-Z7;t*(C}^B-;#Q3)!Y*2{9CkA^ZBW zR@N|5V;f6C<26|ZGq&&<^~3is_}-7l{kZP?ah>}*=RD7Kp66V54A#VepXWFa008_( zhWgh50K|C&0WfaPVB`JPg)l5-zpPk+6|egA%07-~h(I`li3`y>f1*Qs}@==xJjjnUlP zPY5~DVnY9g&R#8_tP_SFL<2N>M&XIz@fIR&ysWSBP_$9q&j}y4P0-SkTKjs(&X0sL zc1=@VUobh!C;ZKba1PnG5*`=>4^${4lSn!Rq#~KS0{hUFqD~ew>MMT@+k1ZCW^48~NPx=opv!8xO2*;9&6>JxP|A^Yv#vY|soq`_+&orL2- zDC2RjZ9>@TkOA{7mJ?4oNYrzjDX2+_M`{|#i}@-Rw0_&A(AkK!Q)vlpdfX~Ta2!#D zW>_$?H~q6u5i#?C!5c9JQYI;}488owL%T;q%-GT?9S)jL;lh-1|7C(mO5-;6cky^M zYk*4;76$D^)4RfNQ&k-~dB# znF@t_n|Rrq+=g9SUo9dmJwE zaC8?2lx|2B@E>77kKC2vf)hi}z~G}D59+r3Y&Tan=7H~e8$?iXA=K~w!=sLVce3%| z{e>bxfyAnWi8g?6RTGPo(Jriql2$l9E~ETzH_u{ALVZ5}~k z$}NAvbrIUyg3e=>5$QI{A06qB>N#)qAMcOvAZ2tF6Wt!t=n1IKYO(plI8dP>fs`rg zm~{5uta1eWyB>F?IntlXdtLxgyQRj>0f_LACeW#xqy)?soDqXW^WD|5=ekajUJwEA z1M+l@FdHmn0lh|H!Nl64`0n;<5~VXU)T*oSdMJ(uSSC_at!`Ir?gK~ozd2CPXnX~b zeb$q2AA^5~L1?j&3xLCzX9OIKeftV3rxwiZAx$~9?_16VxF*9-`qY6k+|j@ABty07 zvm$B-)H~|W3TmRcD=p!c6U`6L5yKt-x*jlPxgHdJ4ofX47UA08$gx|jkl&XIAfJQ| z{54)da~acA)|G%RK2j_K*%#quk`B!*3wiQ%$J!ZTvUQ;(*fUUZHI&>Om$=hhcD5A0 z)jt;>&Vh;u{{rsc8tKjAo$~jvAoJ;ia)GLPWT_@&E@7c`6Xf7@ouWDyzrO*A@VO!T z5}Fg!lm>b9xOEzZK=ELU=&eZ)?mpVy3jxy7gg45#j`MO1K&=gcQ{c6-WrMjEf;~K3 zf*^hF%Hzn=xckf-Q@vllj?2=IuRhUR2826Z>8+=f61DuqXX;Q{$bum=R%xRCfy4ZRNJ_t-p3#3G0%u$Ay;`L_dW8w{(HTi%;Hw_ zsvZ*W_f~BZB*Cslq!Xa!trfVg-eilqllPpxVP(4?Cjc1o>v^zR1IqK!%B zS6rV7Qy5$2(oD;Qkb!Eu>Y1iPYrWc?H-{}hYM#JKX%VICYqqYccS`jdx@KIiz^B2t z<`;7$E(FSlE~E2S33n9As&N$K<2kI9+RGiuNtx3&Gu(`vuV4MJ#3UKI@OT|$e@pb` zlf<6-4mfC)TmO=HHT1E;jnNI^aPa@XtL5{{M*jrfm3EFb{8S0qq~=_dG-jEkBTTvN zR8r4VTvPj<4N`RP_0xMjmV@JU9WYNwMH*uISLG)qQab)HZW3RqwjQ}=6E(DrvL}&T zA3O7lJEl_*8)rea^=7%YLahPUaL(&vTNP* zFpo1!C>Rov0t)D?2%fmJ04b?f8Ee+VSmZ8a{j2(bo?{Og`f8pY?iNDPtcXRCo9?UP z{=_fF`cz(w#<%6{Pe&i$iPB%8s_@o3>Unc2ugANQQ*~a z@mC?B%DUlWChm`|<|`$$$UhsN^_?#vtJLnK`VVHK(?~0UE#4uS`8G|PT(ptMF!Fgd z!7%8z&K4)NuOoxW;eGNvqed+%FyswsFxm77V`RKaS(c;zWd@`*ywHV^w~$`nVEkNt zK~$D*Hg}2>fdBl!Ilk$}{96e_7mFYni+!JVfVn22j)#7UYj)DrdF- zLgnQWti?Xy5Gc-SK~q+W$3Ftgm8?k2{#x$>y{?G$ggAmT@VOgU+?{R$*{7GCv84{V z5E}c3?#);P)*&@Nk0|Xppsu4S1cu}mQ^jC11y&! zH)YVyXk&V4oSG&4)c`PM)#Mr?zMpG%FCt5||I?Mr*aZaFy#E%DO5_G=GLsFUF(&0r ze~`K;0iVWb4~l@&>oq4|F5vR?o3y51N~cxe+$WomgSP= zIs?yjw;OX%1-Wlk+B>N7#rT`y$~||{ zD*BGSrhp{nnV=?YaFulER>-1@J~iDGlX5ZwCeHD3wWR{r+nd=(-p~yFp#Mt)5eEI5 z0X3})xNW`eDCkx4tV{f3pong?AP3v<`yb95yZ%##JH{LROXHs~s0sv*qbM~r4Qknu zeQ(*|2&j|MorF`nst$lboGEd8qsz-Xs0`Yx#F6bpH@$6C3w33|lCb3_=c2cJ3N+V|_u`^5*2e68WZ&Zx?e9@P#z1I)yHxZM@`r+S{gyi1^& z%ZwGM!}90kfR9^;c(#0K59z#1fVoEMk(03hFtnK`V@ncLdnXhnW*x|>O7yJ(u<{bG z@;QwtHz?i@BrOq4O2HBb8N0zP(7LvIG2l#iIr|#I`IWEuaksP6P&a-E-U2^V`N*D2 zz)e>Wm~DY>M&QqmcJ_*nwT^}WaaNT_%c|Yf0hL*l`#Pvp+Fj}G>^6|{gIo+n9n|d6 zc`4-l{dsd0Ta=(O#pxR#HR7-RQ*eQ9H70;o-XKlB8v;m^GF5cv)>dCr`UBZ~;l3WEDBOh13FHTx9 z==<5HnI$_>cyzdy=LkOamD*AeYbM-o+Nzmu6=GQ=)tKYAiL&5V^pkjX-|6f&DfMt{ z`~9duH&jmrQ!3td$40`v`RL^7AZ8;=TAN-FlQ&+mgtZfaTP7^|eET_Zr#Ku>49O$8 zZ_$RUvS$C=FsAO@1S_pHauN*86zwjW5A~q`SPd~@TGVgHn*Hy`QbJ96pTcP2#E+IY zOPG{VC{OP70F?1|QfvM$sEmB3o|t^Zk4yE*mJi0~dmrC&boz zWKZPM!|fR5Qmx-Y18)1^DevD*X}nRj1ytTNXV6My$%*GJcZYAjMBt-VRsv9d|I-LZ ztTl4dVCRyDIYX8nWdnJ+flDPXVc)hc=s>h;5JJb+zAOsGJVj&rE7*uH-TqTgtbW&odQ{Ya2B-l$!~SW1rA<+9cpV6r6+6AtfJQeDgCLIg`EM{*{-= z>%%WEVKk`mi-|Lj|FjJ}^-Yr=OTzleE1VP2vp)Cy6M9bSgG~dny|~ldPU=0Qa)xq0 zF*XM0Sctyl0K}RsuhF^Z!vOg)%&F#~EX|9`v#)4vN z-tL~L;!I9uW0;wKH;J{wV6sXdthMOy_Sv!BQV;tsi*{cv&+6(k>l0UHiK@=OU9XQw z8!qan9g(@w1Mc^}W{O#p-@69Cy%wb?uF0q&YlT+2AvNj#19CxHB8|wJ9A=V(so6=W z(U9Ngt%^SVw`m;OYVOL(!n-xwE$BBow3!UosjyERwM@Q~|)fH}*FY-}LlCp6?s-~7}nH>A8 zk?>ZzW_OTwu;__vL9zEiGP8_w@&k5|40?K}$1SyfQZg6V< zM&9`{?N=?4h(ZJ#mUaO=Fu`ls!vOUdxFSE2`d```wq9|(;$=g2TmZkxW3^P+4W){f zGA;;9MOHu_rWN1?-jbbmVloC_j=x7~W6!N-Jw7OCKamy$n&ourvu8VQH?Yq}*Z=Fn zJ+X9{V7b>pB=6}U+$G_Cm4D6gUMOeoo1W!7|COg|;| zmr}_EP9E2C$&AGq6K@5UW|dP1z?#vyCZh7v4O}gD099Qk);Y7*cY5l1n!-Wfi!tr0 z)(v%tw=b87CLQ6d$V{(G@^WrpF!0(4REqVHWiB4sKon!z#5R(vOtJsy!5DG3;VYI+iSAfzfym8FZ{KC-7 z+2{F;?ZZCrN&m|1|Kkv4lo|`9;EO5@(3;4A`Op%+8{K}<)xTQP?|DvHArLc}ryn`} z?pwUV1<@KBJ!c3_GuWvXng6#v&(uV_lYdqqJTAYK*`b1K9+=RG-eRGMHuvC#<>bQUi(|F7o>vKBDH(AZj`!|ltruK5 z7Nh{N^{@wgU(mi=X(Lm;tUFLS^Ft!bjBrl{z>(rRd`*gYl^Z!+KEtcmPjw0a-gq#D-L8)@z` z9{3+jAMi~-Yx;Js{3>}e{)jh@*dx<9ACHRi^ZXfgxu#KU<6AE&u=k literal 0 HcmV?d00001 diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index dfa35de1b34..8e343e1a6e0 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -9,6 +9,7 @@ import langsmithLogo from "../../public/assets/logos/langsmith.png"; import newrelicLogo from "../../public/assets/logos/newrelic.png"; import openmeterLogo from "../../public/assets/logos/openmeter.png"; import otelLogo from "../../public/assets/logos/otel.png"; +import pointfiveLogo from "../../public/assets/logos/pointfive.png"; interface CallbackConfig { id: string; @@ -162,6 +163,17 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ }, description: "OpenTelemetry Logging Integration", }, + { + id: "pointfive", + displayName: "PointFive", + logo: pointfiveLogo.src, + supports_key_team_logging: false, + dynamic_params: { + POINTFIVE_API_KEY: "password", + POINTFIVE_API_URL: "text", + }, + description: "PointFive Logging Integration", + }, { id: "s3", displayName: "S3", From 123c01c4395ef5225e04571436268efc0a14766a Mon Sep 17 00:00:00 2001 From: Yinon Kahta Date: Thu, 27 Aug 2026 14:12:22 +0300 Subject: [PATCH 308/411] feat(pointfive): answer the ui health check with a liveness ping --- .../health_endpoints/_health_endpoints.py | 25 +++++- .../health_endpoints/test_health_endpoints.py | 86 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 747cfa526d9..623e6ffbc87 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -209,6 +209,7 @@ services = ( "arize", "galileo", "newrelic", + "pointfive", "sqs", ] | str @@ -296,6 +297,7 @@ async def health_services_endpoint( "arize", "galileo", "newrelic", + "pointfive", "sqs", ]: raise HTTPException( @@ -320,7 +322,7 @@ async def health_services_endpoint( service == "openmeter" or service == "braintrust" or service == "generic_api" - or (service_in_success_callbacks and service != "langfuse") + or (service_in_success_callbacks and service not in ("langfuse", "pointfive")) ): _ = await litellm.acompletion( model="openai/litellm-mock-response-model", @@ -412,6 +414,27 @@ async def health_services_endpoint( ), } + elif service == "pointfive": + if not _is_proxy_admin(user_api_key_dict): + non_admin_detail: Final[_ServiceTestErrorDetail] = { + "error": "Only proxy admins can trigger the PointFive liveness ping." + } + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=non_admin_detail) + from litellm.integrations.pointfive import PointFiveLogger + + try: + pointfive_logger: Final = PointFiveLogger(start_periodic_flush=False) + except ValueError as missing_key: + # No key configured is the answer the operator asked for, not a server error. + no_key: Final[_ServiceTestSuccessResponse] = {"status": "unhealthy", "message": str(missing_key)} + return no_key + response = await pointfive_logger.async_health_check() + pointfive_health: Final[_ServiceTestSuccessResponse] = { + "status": response["status"], + "message": (response["error_message"] if response["status"] == "unhealthy" else "PointFive is healthy") + or "PointFive is healthy", + } + return pointfive_health if service == "webhook": user_info: Final = CallInfo( token=user_api_key_dict.token or "", diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index b06d87ac67e..527d46931fe 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -1058,6 +1058,35 @@ async def test_health_services_endpoint_galileo(status, error_message): mock_instance.async_health_check.assert_awaited_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status,error_message", + [ + ("healthy", ""), + ("unhealthy", "PointFive authentication failed"), + ], +) +async def test_health_services_endpoint_pointfive(monkeypatch, status, error_message): + import litellm.integrations.pointfive as pointfive_package + + mock_instance = MagicMock() + mock_instance.async_health_check = AsyncMock(return_value={"status": status, "error_message": error_message}) + logger_class = MagicMock(return_value=mock_instance) + monkeypatch.setattr(pointfive_package, "PointFiveLogger", logger_class) + + result = await health_services_endpoint(user_api_key_dict=_pointfive_admin(), service="pointfive") + + if status == "healthy": + assert result["status"] == "healthy" + assert result["message"] == "PointFive is healthy" + else: + assert result["status"] == "unhealthy" + assert result["message"] == error_message + mock_instance.async_health_check.assert_awaited_once() + # A check that left the periodic flush running would leak a flusher per press of the ui test button. + logger_class.assert_called_once_with(start_periodic_flush=False) + + @pytest.mark.asyncio async def test_health_services_endpoint_datadog_llm_observability(): """ @@ -3131,3 +3160,60 @@ def test_test_model_connection_accepts_image_edit_mode(monkeypatch): assert response.status_code == 200, response.text assert response.json()["status"] == "success" + + +def _pointfive_admin() -> UserAPIKeyAuth: + return UserAPIKeyAuth(token="admin-token", user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) + + +@pytest.mark.asyncio +async def test_health_services_endpoint_pointfive_without_a_key_is_unhealthy_not_a_server_error(monkeypatch): + """ + The logger refuses to start without an api key. + + That refusal is the answer the operator asked for, so it has to come back as an + unhealthy result rather than a 500 from the endpoint. + """ + import litellm.integrations.pointfive as pointfive_package + + def refuse(**_): + raise ValueError("pointfive logging requires an api key. Set POINTFIVE_API_KEY") + + monkeypatch.setattr(pointfive_package, "PointFiveLogger", refuse) + + result = await health_services_endpoint(user_api_key_dict=_pointfive_admin(), service="pointfive") + + assert result["status"] == "unhealthy" + assert "requires an api key" in result["message"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "role", + [ + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + LitellmUserRoles.TEAM, + LitellmUserRoles.CUSTOMER, + ], +) +async def test_health_services_endpoint_pointfive_blocks_non_admin(monkeypatch, role): + """ + The ping travels on the proxy-wide PointFive credential and stamps liveness at PointFive. + + A tenant key must not be able to keep an integration looking alive, or read back + account-level authentication failures through it. + """ + import litellm.integrations.pointfive as pointfive_package + from litellm.proxy._types import ProxyException + + logger_class = MagicMock() + monkeypatch.setattr(pointfive_package, "PointFiveLogger", logger_class) + + with pytest.raises(ProxyException) as raised: + await health_services_endpoint( + user_api_key_dict=UserAPIKeyAuth(token="t", user_id="u", user_role=role), service="pointfive" + ) + + assert str(raised.value.code) == "403" + logger_class.assert_not_called() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6d62ce2b675..ea613f54af2 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -49399,7 +49399,7 @@ export interface operations { parameters: { query: { /** @description Specify the service being hit. */ - service: ("slack_budget_alerts" | "langfuse" | "langfuse_otel" | "slack" | "ms_teams" | "openmeter" | "webhook" | "email" | "braintrust" | "datadog" | "datadog_llm_observability" | "generic_api" | "arize" | "galileo" | "newrelic" | "sqs") | string; + service: ("slack_budget_alerts" | "langfuse" | "langfuse_otel" | "slack" | "ms_teams" | "openmeter" | "webhook" | "email" | "braintrust" | "datadog" | "datadog_llm_observability" | "generic_api" | "arize" | "galileo" | "newrelic" | "pointfive" | "sqs") | string; }; header?: never; path?: never; From 134d1f3899ebb9f11415e1917c27d372f9e98960 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 10 Sep 2026 13:17:14 +0000 Subject: [PATCH 309/411] fix(model_prices): registry audit 2026-09-10, absorb open pricing PRs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 60 +++++++++++++++--- model_prices_and_context_window.json | 60 +++++++++++++++--- .../llms/openai/test_gpt5_transformation.py | 63 +++++++++++-------- 3 files changed, 137 insertions(+), 46 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0d2eda93323..d8160a3d842 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -30295,7 +30295,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": false }, "gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -30340,7 +30340,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": false }, "gpt-5.1-chat-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -30432,7 +30432,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": false }, "gpt-5.2-2025-12-11": { "cache_read_input_token_cost": 1.75e-07, @@ -30478,7 +30478,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": false }, "gpt-5.2-chat-latest": { "cache_read_input_token_cost": 1.75e-07, @@ -31474,7 +31474,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 @@ -31526,7 +31526,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 @@ -31578,7 +31578,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, @@ -31629,7 +31629,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, @@ -33709,13 +33709,14 @@ "supports_tool_choice": true }, "jina-reranker-v2-base-multilingual": { - "input_cost_per_token": 1.8e-08, + "input_cost_per_token": 5e-08, "litellm_provider": "jina_ai", "max_input_tokens": 1024, "max_output_tokens": 1024, "max_tokens": 1024, "mode": "rerank", - "output_cost_per_token": 1.8e-08 + "output_cost_per_token": 0.0, + "source": "https://api.jina.ai/v1/models" }, "jp.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -39969,6 +39970,45 @@ "supports_tool_choice": true, "supports_vision": true }, + "openrouter/openai/gpt-5.6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "default_reasoning_effort": "medium", + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "source": "https://openrouter.ai/openai/gpt-5.6-sol", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/openai/gpt-oss-120b": { "input_cost_per_token": 3.7e-08, "litellm_provider": "openrouter", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0d2eda93323..d8160a3d842 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30295,7 +30295,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": false }, "gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -30340,7 +30340,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": false }, "gpt-5.1-chat-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -30432,7 +30432,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": false }, "gpt-5.2-2025-12-11": { "cache_read_input_token_cost": 1.75e-07, @@ -30478,7 +30478,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": false }, "gpt-5.2-chat-latest": { "cache_read_input_token_cost": 1.75e-07, @@ -31474,7 +31474,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 @@ -31526,7 +31526,7 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 @@ -31578,7 +31578,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, @@ -31629,7 +31629,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, @@ -33709,13 +33709,14 @@ "supports_tool_choice": true }, "jina-reranker-v2-base-multilingual": { - "input_cost_per_token": 1.8e-08, + "input_cost_per_token": 5e-08, "litellm_provider": "jina_ai", "max_input_tokens": 1024, "max_output_tokens": 1024, "max_tokens": 1024, "mode": "rerank", - "output_cost_per_token": 1.8e-08 + "output_cost_per_token": 0.0, + "source": "https://api.jina.ai/v1/models" }, "jp.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -39969,6 +39970,45 @@ "supports_tool_choice": true, "supports_vision": true }, + "openrouter/openai/gpt-5.6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "default_reasoning_effort": "medium", + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "source": "https://openrouter.ai/openai/gpt-5.6-sol", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/openai/gpt-oss-120b": { "input_cost_per_token": 3.7e-08, "litellm_provider": "openrouter", diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index c86ce4df2ac..b538fad71a2 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -276,6 +276,17 @@ def test_gpt5_drops_reasoning_effort_xhigh_when_requested(config: OpenAIConfig): assert "reasoning_effort" not in params +def test_gpt5_1_gpt5_2_gpt5_4_drop_minimal_reasoning_effort(config: OpenAIConfig): + for model in ("gpt-5.1", "gpt-5.2", "gpt-5.4", "gpt-5.4-pro"): + params = config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model=model, + drop_params=True, + ) + assert "reasoning_effort" not in params + + # GPT-5.1 temperature handling tests def test_gpt5_1_model_detection(gpt5_config: OpenAIGPT5Config): """Test that models supporting reasoning_effort='none' are correctly detected via model map.""" @@ -388,26 +399,26 @@ def test_gpt5_4_mini_allows_reasoning_effort_none(config: OpenAIConfig): assert params["reasoning_effort"] == "none" -def test_gpt5_4_allows_reasoning_effort_minimal(config: OpenAIConfig): - """gpt-5.4 supports reasoning_effort='minimal'.""" - params = config.map_openai_params( - non_default_params={"reasoning_effort": "minimal"}, - optional_params={}, - model="gpt-5.4", - drop_params=False, - ) - assert params["reasoning_effort"] == "minimal" +def test_gpt5_4_rejects_reasoning_effort_minimal(config: OpenAIConfig): + """gpt-5.4 rejects reasoning_effort='minimal'.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) -def test_gpt5_4_pro_allows_reasoning_effort_minimal(config: OpenAIConfig): - """gpt-5.4-pro supports reasoning_effort='minimal'.""" - params = config.map_openai_params( - non_default_params={"reasoning_effort": "minimal"}, - optional_params={}, - model="gpt-5.4-pro", - drop_params=False, - ) - assert params["reasoning_effort"] == "minimal" +def test_gpt5_4_pro_rejects_reasoning_effort_minimal(config: OpenAIConfig): + """gpt-5.4-pro rejects reasoning_effort='minimal'.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model="gpt-5.4-pro", + drop_params=False, + ) def test_gpt5_4_mini_rejects_reasoning_effort_minimal(config: OpenAIConfig): @@ -468,13 +479,13 @@ def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): def test_gpt5_minimal_dict_accepted_for_supported_model(config: OpenAIConfig): - """Dict with effort='minimal' passes through for gpt-5.4+.""" + """Dict with effort='minimal' passes through for gpt-5.""" params = config.map_openai_params( non_default_params={ "reasoning_effort": {"effort": "minimal", "summary": "detailed"} }, optional_params={}, - model="gpt-5.4", + model="gpt-5", drop_params=False, ) assert params["reasoning_effort"] == "minimal" @@ -482,8 +493,8 @@ def test_gpt5_minimal_dict_accepted_for_supported_model(config: OpenAIConfig): def test_gpt5_supports_reasoning_effort_level_minimal(gpt5_config: OpenAIGPT5Config): """Test that _supports_reasoning_effort_level correctly identifies minimal support.""" - assert gpt5_config._supports_reasoning_effort_level("gpt-5.4", "minimal") - assert gpt5_config._supports_reasoning_effort_level("gpt-5.4-pro", "minimal") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4", "minimal") + assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-pro", "minimal") assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-mini", "minimal") assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-nano", "minimal") @@ -504,10 +515,10 @@ def test_gpt5_minimal_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( "openai/gpt-5.4-mini", "minimal" ) - assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled( + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( "gpt-5.4", "minimal" ) - assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled( + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( "gpt-5.4-pro", "minimal" ) @@ -523,8 +534,8 @@ def test_is_explicitly_disabled_factory_minimal(): assert _is_explicitly_disabled_factory("gpt-5.4-mini", None, key) assert _is_explicitly_disabled_factory("gpt-5.4-nano", None, key) assert _is_explicitly_disabled_factory("openai/gpt-5.4-mini", None, key) - assert not _is_explicitly_disabled_factory("gpt-5.4", None, key) - assert not _is_explicitly_disabled_factory("gpt-5.4-pro", None, key) + assert _is_explicitly_disabled_factory("gpt-5.4", None, key) + assert _is_explicitly_disabled_factory("gpt-5.4-pro", None, key) assert not _is_explicitly_disabled_factory("gpt-5.4-turbo-preview", None, key) From 996ca7c5b2bb3f7adee1ef56756f51d4ec0759dc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:04:19 -0700 Subject: [PATCH 310/411] test(cost): assert jina rerank spend at the registry rate --- litellm/llms/jina_ai/rerank/transformation.py | 3 --- tests/test_litellm/test_cost_calculator.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/litellm/llms/jina_ai/rerank/transformation.py b/litellm/llms/jina_ai/rerank/transformation.py index a8f3388d092..2a81c38fe34 100644 --- a/litellm/llms/jina_ai/rerank/transformation.py +++ b/litellm/llms/jina_ai/rerank/transformation.py @@ -157,9 +157,6 @@ class JinaAIRerankConfig(BaseRerankConfig): billed_units: RerankBilledUnits | None = None, model_info: ModelInfo | None = None, ) -> tuple[float, float]: - """ - Jina AI reranker is priced at $0.000000018 per token. - """ if ( model_info is None or "input_cost_per_token" not in model_info diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 8f8a7640c08..f610821e06a 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -18,6 +18,7 @@ from litellm.cost_calculator import ( response_cost_calculator, ) from litellm.types.llms.openai import OpenAIRealtimeStreamList +from litellm.types.rerank import RerankResponse from litellm.types.utils import ( CacheCreationTokenDetails, ModelInfo, @@ -128,6 +129,22 @@ def test_completion_cost_uses_response_model_for_dynamic_routing(_local_model_co assert cost > 0, "Cost should be calculated using response model" +def test_jina_rerank_bills_total_tokens_at_input_rate_only(_local_model_cost_map): + response: Final = RerankResponse( + id="rerank-1", + results=[{"index": 0, "relevance_score": 0.9}], + meta={"billed_units": {"total_tokens": 1000}}, + ) + + cost: Final = completion_cost( + completion_response=response, + model="jina_ai/jina-reranker-v2-base-multilingual", + call_type="rerank", + ) + + assert cost == pytest.approx(1000 * 5e-08) + + def test_cost_calculator_with_response_cost_in_additional_headers(): class MockResponse(BaseModel): _hidden_params = { From 1273e46b8bf4821643eb4b948517c95de34bf632 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 15:45:25 -0400 Subject: [PATCH 311/411] feat(redis): add ElastiCache IAM authentication --- litellm/_redis.py | 57 ++++- litellm/_redis_credential_provider.py | 85 +++++++- litellm/proxy/_types.py | 4 + .../cache_settings_endpoints.py | 36 ++++ .../coordination_redis_endpoints.py | 29 +++ tests/test_litellm/test_redis.py | 196 +++++++++++++++++- .../test_redis_credential_provider.py | 122 +++++++++++ 7 files changed, 520 insertions(+), 9 deletions(-) create mode 100644 tests/test_litellm/test_redis_credential_provider.py diff --git a/litellm/_redis.py b/litellm/_redis.py index 3e68d50cf16..e289cc3b6c3 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -24,6 +24,7 @@ from redis.credentials import CredentialProvider from litellm import get_secret, get_secret_str from litellm._redis_credential_provider import ( AzureADCredentialProvider, + ElastiCacheIAMCredentialProvider, GCPIAMCredentialProvider, _generate_gcp_iam_access_token, ) @@ -75,6 +76,10 @@ def _get_redis_kwargs(): "azure_client_id", "azure_tenant_id", "azure_client_secret", + "aws_iam_auth", + "aws_iam_user_name", + "aws_iam_cache_name", + "aws_iam_region", } available_args: Final = {x for x in _unwrapped_init_args(redis.Redis) if x not in exclude_args} | include_args @@ -270,6 +275,32 @@ def _redis_kwargs_from_environment(): return return_dict +def _is_true(value: object | None) -> bool: + return value is True or (isinstance(value, str) and value.lower() == "true") + + +def _build_elasticache_iam_provider(redis_kwargs: dict) -> ElastiCacheIAMCredentialProvider | None: + if not _is_true(redis_kwargs.get("aws_iam_auth")): + return None + + required_settings: Final = { + "aws_iam_user_name": redis_kwargs.get("aws_iam_user_name"), + "aws_iam_cache_name": redis_kwargs.get("aws_iam_cache_name"), + "aws_iam_region": redis_kwargs.get("aws_iam_region") + or get_secret_str("AWS_REGION") + or get_secret_str("AWS_DEFAULT_REGION"), + } + missing_settings: Final = tuple(name for name, value in required_settings.items() if not value) + if missing_settings: + raise ValueError("AWS ElastiCache IAM Redis authentication requires: " + ", ".join(missing_settings)) + + return ElastiCacheIAMCredentialProvider( + user_name=str(required_settings["aws_iam_user_name"]), + cache_name=str(required_settings["aws_iam_cache_name"]), + region=str(required_settings["aws_iam_region"]), + ) + + def create_gcp_iam_redis_connect_func( service_account: str, ssl_ca_certs: str | None = None, @@ -540,6 +571,7 @@ def _get_redis_client_logic(**env_overrides): _azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN") _azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true" + _aws_iam_enabled: Final = _is_true(redis_kwargs.get("aws_iam_auth")) if _azure_ad_enabled and _gcp_service_account is not None: verbose_logger.warning( @@ -560,13 +592,24 @@ def _get_redis_client_logic(**env_overrides): azure_tenant_id=_azure_tenant_id, azure_client_secret=_azure_client_secret, ) - # Marker for async paths to detect Azure AD auth. The live credential - # object is attached separately as `_azure_credential` by - # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret - # are intentionally NOT exposed on the function to avoid leaking - # credentials via inspection or logging. redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True + if _aws_iam_enabled and _gcp_service_account is not None: + verbose_logger.warning( + "Both GCP IAM (gcp_service_account) and AWS ElastiCache IAM (aws_iam_auth) are configured " + "for Redis. Using GCP IAM. Remove one to avoid misconfiguration." + ) + elif _aws_iam_enabled and _azure_ad_enabled: + verbose_logger.warning( + "Both Azure AD (azure_redis_ad_token) and AWS ElastiCache IAM (aws_iam_auth) are configured " + "for Redis. Using Azure AD. Remove one to avoid misconfiguration." + ) + elif _aws_iam_enabled: + aws_provider: Final = _build_elasticache_iam_provider(redis_kwargs) + if aws_provider is not None: + verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") + redis_kwargs["credential_provider"] = aws_provider + redis_kwargs.pop("gcp_service_account", None) redis_kwargs.pop("gcp_ssl_ca_certs", None) @@ -575,6 +618,10 @@ def _get_redis_client_logic(**env_overrides): redis_kwargs.pop("azure_client_id", None) redis_kwargs.pop("azure_tenant_id", None) redis_kwargs.pop("azure_client_secret", None) + redis_kwargs.pop("aws_iam_auth", None) + redis_kwargs.pop("aws_iam_user_name", None) + redis_kwargs.pop("aws_iam_cache_name", None) + redis_kwargs.pop("aws_iam_region", None) if redis_kwargs.get("credential_provider") is not None: redis_kwargs.pop("redis_connect_func", None) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index ba0398789a6..a20bbfc530c 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -1,7 +1,8 @@ import asyncio import threading import time -from typing import Final, Protocol +from typing import Any, Final, Protocol +from urllib.parse import quote from redis.credentials import CredentialProvider @@ -117,6 +118,88 @@ class GCPIAMCredentialProvider(CredentialProvider): return (token,) +_ELASTICACHE_SERVICE_NAME: Final = "elasticache" +_ELASTICACHE_TOKEN_TTL_SECONDS: Final = 900 + + +class _FrozenBotocoreCredentials(Protocol): + access_key: str + secret_key: str + token: str | None + + +class _BotocoreCredentials(Protocol): + def get_frozen_credentials(self) -> _FrozenBotocoreCredentials: ... + + +class _BotocoreCredentialsResolver(Protocol): + def __call__(self) -> _BotocoreCredentials | None: ... + + +class ElastiCacheIAMCredentialProvider(CredentialProvider): + def __init__( + self, + user_name: str, + cache_name: str, + region: str, + credentials_resolver: _BotocoreCredentialsResolver | None = None, + token_lifetime_seconds: int = _ELASTICACHE_TOKEN_TTL_SECONDS, + ) -> None: + self._user_name = user_name + self._cache_name = cache_name + self._region = region + self._credentials_resolver = credentials_resolver or self._resolve_credentials + self._credentials: _BotocoreCredentials | None = None + self._token_lifetime_seconds = token_lifetime_seconds + + @staticmethod + def _resolve_credentials() -> Any: + try: + import botocore.session + except ImportError as e: + raise ImportError( + "botocore is required for ElastiCache IAM Redis authentication. Install it with: pip install boto3" + ) from e + + return botocore.session.get_session().get_credentials() + + def _get_credentials(self) -> tuple[str, str]: + credentials: Final = self._credentials if self._credentials is not None else self._credentials_resolver() + if credentials is None: + raise RuntimeError("Unable to resolve AWS credentials for ElastiCache IAM Redis authentication") + self._credentials = credentials + + frozen_credentials: Final = credentials.get_frozen_credentials() + if frozen_credentials is None: + raise RuntimeError("Unable to resolve AWS credentials for ElastiCache IAM Redis authentication") + + try: + from botocore.auth import SigV4QueryAuth + from botocore.awsrequest import AWSRequest + except ImportError as e: + raise ImportError( + "botocore is required for ElastiCache IAM Redis authentication. Install it with: pip install boto3" + ) from e + + request: Final = AWSRequest( + method="GET", + url=(f"https://{self._cache_name}/?Action=connect&User={quote(self._user_name, safe='')}"), + ) + SigV4QueryAuth( + frozen_credentials, + _ELASTICACHE_SERVICE_NAME, + self._region, + expires=self._token_lifetime_seconds, + ).add_auth(request) + return self._user_name, request.url.removeprefix("https://") + + def get_credentials(self) -> tuple[str, str]: + return self._get_credentials() + + async def get_credentials_async(self) -> tuple[str, str]: + return await asyncio.to_thread(self._get_credentials) + + class AzureADCredentialProvider(CredentialProvider): """ redis.credentials.CredentialProvider implementation that supplies Azure AD diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c22bb76629d..99103ff3706 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2427,6 +2427,10 @@ class CoordinationRedisParams(LiteLLMPydanticObjectBase): ) sentinel_password: str | None = Field(None, description="password for the sentinel nodes") service_name: str | None = Field(None, description="sentinel service name") + aws_iam_auth: bool | str | None = Field(None, description="enable AWS ElastiCache IAM authentication") + aws_iam_user_name: str | None = Field(None, description="AWS ElastiCache IAM user name") + aws_iam_cache_name: str | None = Field(None, description="AWS ElastiCache cache name") + aws_iam_region: str | None = Field(None, description="AWS region for ElastiCache IAM authentication") def has_connection_target(self) -> bool: return any(value is not None for value in (self.host, self.url, self.startup_nodes, self.sentinel_nodes)) diff --git a/litellm/types/management_endpoints/cache_settings_endpoints.py b/litellm/types/management_endpoints/cache_settings_endpoints.py index 32b32991449..10aece6efa4 100644 --- a/litellm/types/management_endpoints/cache_settings_endpoints.py +++ b/litellm/types/management_endpoints/cache_settings_endpoints.py @@ -237,4 +237,40 @@ CACHE_SETTINGS_FIELDS: Final[list[CacheSettingsField]] = [ ui_field_name="SSL Check Hostname", redis_type=None, ), + CacheSettingsField( + field_name="aws_iam_auth", + field_type="Boolean", + field_value=None, + field_description="Enable AWS ElastiCache IAM authentication", + field_default=False, + ui_field_name="AWS IAM Authentication", + redis_type=None, + ), + CacheSettingsField( + field_name="aws_iam_user_name", + field_type="String", + field_value=None, + field_description="AWS ElastiCache IAM user name", + field_default=None, + ui_field_name="AWS IAM User Name", + redis_type=None, + ), + CacheSettingsField( + field_name="aws_iam_cache_name", + field_type="String", + field_value=None, + field_description="AWS ElastiCache cache name", + field_default=None, + ui_field_name="AWS IAM Cache Name", + redis_type=None, + ), + CacheSettingsField( + field_name="aws_iam_region", + field_type="String", + field_value=None, + field_description="AWS region for ElastiCache IAM authentication", + field_default=None, + ui_field_name="AWS IAM Region", + redis_type=None, + ), ] diff --git a/litellm/types/management_endpoints/coordination_redis_endpoints.py b/litellm/types/management_endpoints/coordination_redis_endpoints.py index 30033346ed7..86b32752cbb 100644 --- a/litellm/types/management_endpoints/coordination_redis_endpoints.py +++ b/litellm/types/management_endpoints/coordination_redis_endpoints.py @@ -102,4 +102,33 @@ COORDINATION_REDIS_SETTINGS_FIELDS: Final[list[CoordinationRedisSettingsField]] ui_field_name="Service Name", section="sentinel", ), + CoordinationRedisSettingsField( + field_name="aws_iam_auth", + field_type="Boolean", + field_description="Enable AWS ElastiCache IAM authentication", + field_default=False, + ui_field_name="AWS IAM Authentication", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="aws_iam_user_name", + field_type="String", + field_description="AWS ElastiCache IAM user name", + ui_field_name="AWS IAM User Name", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="aws_iam_cache_name", + field_type="String", + field_description="AWS ElastiCache cache name", + ui_field_name="AWS IAM Cache Name", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="aws_iam_region", + field_type="String", + field_description="AWS region for ElastiCache IAM authentication", + ui_field_name="AWS IAM Region", + section="connection", + ), ] diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index a96e8541e06..c99a3a074fb 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -24,6 +24,7 @@ from litellm._redis import ( ) from litellm._redis_credential_provider import ( AzureADCredentialProvider, + ElastiCacheIAMCredentialProvider, GCPIAMCredentialProvider, _token_cache, ) @@ -78,6 +79,8 @@ def clean_redis_environment(monkeypatch): "REDIS_URL", "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES", + "AWS_REGION", + "AWS_DEFAULT_REGION", *_get_redis_env_kwarg_mapping(), ): monkeypatch.delenv(var, raising=False) @@ -110,6 +113,17 @@ def test_credential_provider_is_not_environment_derived(): assert "credential_provider" not in mapping.values() +def test_aws_iam_settings_are_environment_derived(): + allowed = _get_redis_kwargs() + mapping = _get_redis_env_kwarg_mapping() + + assert {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} <= allowed + assert mapping["REDIS_AWS_IAM_AUTH"] == "aws_iam_auth" + assert mapping["REDIS_AWS_IAM_USER_NAME"] == "aws_iam_user_name" + assert mapping["REDIS_AWS_IAM_CACHE_NAME"] == "aws_iam_cache_name" + assert mapping["REDIS_AWS_IAM_REGION"] == "aws_iam_region" + + def test_sync_direct_preserves_credential_provider_identity(clean_redis_environment): provider = _StubCredentialProvider() @@ -300,6 +314,180 @@ def test_gcp_kwargs_never_survive_client_logic(clean_redis_environment, override assert "gcp_ssl_ca_certs" not in redis_kwargs +def test_aws_iam_environment_settings_install_provider(clean_redis_environment, monkeypatch): + monkeypatch.setenv("REDIS_AWS_IAM_AUTH", "true") + monkeypatch.setenv("REDIS_AWS_IAM_USER_NAME", "iam-user") + monkeypatch.setenv("REDIS_AWS_IAM_CACHE_NAME", "cache.example.com") + monkeypatch.setenv("REDIS_AWS_IAM_REGION", "us-east-1") + + redis_kwargs = _get_redis_client_logic(host="cache.example.com", port=6379) + + assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + + +def test_aws_iam_settings_are_removed_for_url_and_static_credentials(clean_redis_environment): + redis_kwargs = _get_redis_client_logic( + url="rediss://url-user:url-pass@cache.example.com:6380", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + username="static-user", + password="static-password", + ) + + assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + assert redis_kwargs["url"] == "rediss://cache.example.com:6380" + assert "username" not in redis_kwargs + assert "password" not in redis_kwargs + assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + + +@pytest.mark.parametrize("missing", ["aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"]) +def test_aws_iam_missing_setting_fails_closed(clean_redis_environment, missing): + settings = { + "aws_iam_auth": True, + "aws_iam_user_name": "iam-user", + "aws_iam_cache_name": "cache.example.com", + "aws_iam_region": "us-east-1", + } + settings[missing] = None + + with pytest.raises(ValueError, match=missing): + _get_redis_client_logic(host="cache.example.com", port=6379, **settings) + + +@pytest.mark.parametrize("region_var", ["AWS_REGION", "AWS_DEFAULT_REGION"]) +def test_aws_iam_region_falls_back_to_environment(clean_redis_environment, monkeypatch, region_var): + monkeypatch.setenv(region_var, "sa-east-1") + + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + ) + + provider = redis_kwargs["credential_provider"] + assert isinstance(provider, ElastiCacheIAMCredentialProvider) + assert provider._region == "sa-east-1" + + +def test_aws_iam_region_prefers_aws_region_over_default_region(clean_redis_environment, monkeypatch): + monkeypatch.setenv("AWS_REGION", "sa-east-1") + monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-west-1") + + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + ) + + assert redis_kwargs["credential_provider"]._region == "sa-east-1" + + +def test_aws_iam_region_prefers_explicit_over_environment(clean_redis_environment, monkeypatch): + monkeypatch.setenv("AWS_REGION", "sa-east-1") + + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="explicit-region", + ) + + assert redis_kwargs["credential_provider"]._region == "explicit-region" + + +def test_aws_iam_settings_map_to_distinct_provider_fields(clean_redis_environment): + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=True, + aws_iam_user_name="iam-user-value", + aws_iam_cache_name="iam-cache-value", + aws_iam_region="iam-region-value", + ) + + provider = redis_kwargs["credential_provider"] + assert isinstance(provider, ElastiCacheIAMCredentialProvider) + assert provider._user_name == "iam-user-value" + assert provider._cache_name == "iam-cache-value" + assert provider._region == "iam-region-value" + + +@pytest.mark.parametrize("aws_iam_auth", [False, "false"]) +def test_aws_iam_auth_disabled_does_not_install_provider(clean_redis_environment, aws_iam_auth): + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + aws_iam_auth=aws_iam_auth, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert "credential_provider" not in redis_kwargs + assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + + +def test_explicit_provider_wins_over_aws_iam(clean_redis_environment): + provider = _StubCredentialProvider() + + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + credential_provider=provider, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert redis_kwargs["credential_provider"] is provider + assert "aws_iam_auth" not in redis_kwargs + + +def test_gcp_wins_over_aws_iam(clean_redis_environment): + with patch("litellm._redis.create_gcp_iam_redis_connect_func") as mock_gcp: + mock_gcp.return_value = _gcp_marker_callback() + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + gcp_service_account="sa@example.com", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert "credential_provider" not in redis_kwargs + assert redis_kwargs["redis_connect_func"] is mock_gcp.return_value + + +def test_azure_wins_over_aws_iam(clean_redis_environment): + with patch("litellm._redis.create_azure_ad_redis_connect_func") as mock_azure: + mock_azure.return_value = MagicMock() + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + azure_redis_ad_token="true", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert "credential_provider" not in redis_kwargs + assert redis_kwargs["redis_connect_func"] is mock_azure.return_value + + def test_provider_keeps_the_rest_of_the_url_intact(clean_redis_environment): provider = _StubCredentialProvider() @@ -1530,9 +1718,11 @@ def test_async_sentinel_keeps_the_credential_provider_off_the_monitors(markers, "redis_connect_func": SimpleNamespace(**markers), } - with patch("litellm._redis.async_redis.Sentinel") as mock_sentinel_cls: - with patch("litellm._redis._get_redis_client_logic", return_value=redis_kwargs): - get_redis_async_client() + with ( + patch("litellm._redis.async_redis.Sentinel") as mock_sentinel_cls, + patch("litellm._redis._get_redis_client_logic", return_value=redis_kwargs), + ): + get_redis_async_client() sentinel_kwargs = mock_sentinel_cls.call_args[1]["sentinel_kwargs"] assert sentinel_kwargs["password"] == sentinel_password diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py new file mode 100644 index 00000000000..6299d4d42e0 --- /dev/null +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -0,0 +1,122 @@ +import asyncio +from types import SimpleNamespace +from urllib.parse import parse_qs, urlsplit + +import pytest + +from litellm._redis_credential_provider import ( + ElastiCacheIAMCredentialProvider, + _BotocoreCredentials, +) + + +class _FakeCredentials: + def __init__(self, access_key: str) -> None: + self.access_key = access_key + self.secret_key = "synthetic-secret" + self.token = "synthetic-session-token" + + def get_frozen_credentials(self): + return self + + +class _FakeResolver: + def __init__(self, credentials: _BotocoreCredentials | None) -> None: + self.credentials = credentials + self.calls = 0 + + def __call__(self): + self.calls += 1 + return self.credentials + + +class _RotatingFakeCredentials: + def __init__(self) -> None: + self.calls = 0 + + def __bool__(self) -> bool: + return False + + def get_frozen_credentials(self): + self.calls += 1 + return SimpleNamespace( + access_key=f"AKIA-SYNTHETIC-{self.calls}", + secret_key="synthetic-secret", + token="synthetic-session-token", + ) + + +def test_elasticache_provider_signs_expected_query(): + resolver = _FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=resolver, + ) + + user_name, token = provider.get_credentials() + parsed = urlsplit("https://" + token) + query = parse_qs(parsed.query) + + assert user_name == "iam-user" + assert parsed.netloc == "cache.example.com" + assert query["Action"] == ["connect"] + assert query["User"] == ["iam-user"] + assert query["X-Amz-Expires"] == ["900"] + assert "elasticache" in query["X-Amz-Credential"][0] + assert query["X-Amz-Credential"][0].split("/")[2] == "us-east-1" + assert not token.startswith("https://") + + +def test_elasticache_provider_resolves_credentials_once_but_refreshes_signature(): + rotating_credentials = _RotatingFakeCredentials() + resolver = _FakeResolver(rotating_credentials) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=resolver, + ) + + first = provider.get_credentials() + second = provider.get_credentials() + async_result = asyncio.run(provider.get_credentials_async()) + + assert first[0] == second[0] == async_result[0] == "iam-user" + assert first[1] != second[1] + assert async_result[1] != second[1] + assert resolver.calls == 1 + assert rotating_credentials.calls == 3 + + +def test_elasticache_provider_reports_missing_credentials(): + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=_FakeResolver(None), + ) + + with pytest.raises(RuntimeError, match="Unable to resolve AWS credentials"): + provider.get_credentials() + + +def test_elasticache_provider_recovers_after_a_failed_resolution(): + resolver = _FakeResolver(None) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=resolver, + ) + + with pytest.raises(RuntimeError, match="Unable to resolve AWS credentials"): + provider.get_credentials() + + resolver.credentials = _FakeCredentials("AKIA-SYNTHETIC") + user_name, token = provider.get_credentials() + + assert user_name == "iam-user" + assert token + assert resolver.calls == 2 From 604b9edc533ec236eaa3563e61e5fade4bfd6279 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 15:58:02 -0400 Subject: [PATCH 312/411] test(redis): cover AWS IAM provider install on async cluster nodes No existing test combined aws_iam_auth with startup_nodes, the shape ElastiCache/Valkey Serverless deployments actually use. --- tests/test_litellm/test_redis.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index c99a3a074fb..21c23ba7866 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -488,6 +488,20 @@ def test_azure_wins_over_aws_iam(clean_redis_environment): assert redis_kwargs["redis_connect_func"] is mock_azure.return_value +def test_async_cluster_installs_aws_iam_provider(clean_redis_environment): + startup_nodes = [{"host": "cluster-node", "port": 6379}] + + client = get_redis_async_client( + startup_nodes=startup_nodes, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert isinstance(client.connection_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + + def test_provider_keeps_the_rest_of_the_url_intact(clean_redis_environment): provider = _StubCredentialProvider() From f66891d80f4c7ce4d47a5ddb4ce499e5df21065a Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 19:19:01 -0400 Subject: [PATCH 313/411] fix(redis): type ElastiCache IAM configuration Generated with AI Co-Authored-By: Claude Code --- litellm/_redis.py | 41 ++++++++++--------- litellm/_redis_credential_provider.py | 30 ++++++-------- .../test_redis_credential_provider.py | 25 +++++------ ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 +++++++++ 4 files changed, 65 insertions(+), 51 deletions(-) diff --git a/litellm/_redis.py b/litellm/_redis.py index e289cc3b6c3..2fd8a7100e3 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -279,25 +279,24 @@ def _is_true(value: object | None) -> bool: return value is True or (isinstance(value, str) and value.lower() == "true") -def _build_elasticache_iam_provider(redis_kwargs: dict) -> ElastiCacheIAMCredentialProvider | None: - if not _is_true(redis_kwargs.get("aws_iam_auth")): - return None - - required_settings: Final = { - "aws_iam_user_name": redis_kwargs.get("aws_iam_user_name"), - "aws_iam_cache_name": redis_kwargs.get("aws_iam_cache_name"), - "aws_iam_region": redis_kwargs.get("aws_iam_region") - or get_secret_str("AWS_REGION") - or get_secret_str("AWS_DEFAULT_REGION"), - } - missing_settings: Final = tuple(name for name, value in required_settings.items() if not value) +def _build_elasticache_iam_provider( + user_name: object | None, + cache_name: object | None, + region: object | None, +) -> ElastiCacheIAMCredentialProvider: + required_settings: Final = ( + ("aws_iam_user_name", user_name), + ("aws_iam_cache_name", cache_name), + ("aws_iam_region", region), + ) + missing_settings: Final = tuple(name for name, value in required_settings if not value) if missing_settings: raise ValueError("AWS ElastiCache IAM Redis authentication requires: " + ", ".join(missing_settings)) return ElastiCacheIAMCredentialProvider( - user_name=str(required_settings["aws_iam_user_name"]), - cache_name=str(required_settings["aws_iam_cache_name"]), - region=str(required_settings["aws_iam_region"]), + user_name=str(user_name), + cache_name=str(cache_name), + region=str(region), ) @@ -605,10 +604,14 @@ def _get_redis_client_logic(**env_overrides): "for Redis. Using Azure AD. Remove one to avoid misconfiguration." ) elif _aws_iam_enabled: - aws_provider: Final = _build_elasticache_iam_provider(redis_kwargs) - if aws_provider is not None: - verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") - redis_kwargs["credential_provider"] = aws_provider + verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") + redis_kwargs["credential_provider"] = _build_elasticache_iam_provider( + user_name=redis_kwargs.get("aws_iam_user_name"), + cache_name=redis_kwargs.get("aws_iam_cache_name"), + region=redis_kwargs.get("aws_iam_region") + or get_secret_str("AWS_REGION") + or get_secret_str("AWS_DEFAULT_REGION"), + ) redis_kwargs.pop("gcp_service_account", None) redis_kwargs.pop("gcp_ssl_ca_certs", None) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index a20bbfc530c..85759af2a88 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -1,11 +1,19 @@ +from __future__ import annotations + import asyncio import threading import time -from typing import Any, Final, Protocol +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Final, Protocol from urllib.parse import quote from redis.credentials import CredentialProvider +if TYPE_CHECKING: + from botocore.credentials import Credentials +else: + Credentials = Any # rebind-ok: runtime alias for the type-checking-only botocore import + # Azure AD scope for Redis Cache for Azure. AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" @@ -122,38 +130,24 @@ _ELASTICACHE_SERVICE_NAME: Final = "elasticache" _ELASTICACHE_TOKEN_TTL_SECONDS: Final = 900 -class _FrozenBotocoreCredentials(Protocol): - access_key: str - secret_key: str - token: str | None - - -class _BotocoreCredentials(Protocol): - def get_frozen_credentials(self) -> _FrozenBotocoreCredentials: ... - - -class _BotocoreCredentialsResolver(Protocol): - def __call__(self) -> _BotocoreCredentials | None: ... - - class ElastiCacheIAMCredentialProvider(CredentialProvider): def __init__( self, user_name: str, cache_name: str, region: str, - credentials_resolver: _BotocoreCredentialsResolver | None = None, + credentials_resolver: Callable[[], Credentials | None] | None = None, token_lifetime_seconds: int = _ELASTICACHE_TOKEN_TTL_SECONDS, ) -> None: self._user_name = user_name self._cache_name = cache_name self._region = region self._credentials_resolver = credentials_resolver or self._resolve_credentials - self._credentials: _BotocoreCredentials | None = None + self._credentials: Credentials | None = None self._token_lifetime_seconds = token_lifetime_seconds @staticmethod - def _resolve_credentials() -> Any: + def _resolve_credentials() -> Credentials | None: try: import botocore.session except ImportError as e: diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index 6299d4d42e0..de4afb7c313 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -4,10 +4,7 @@ from urllib.parse import parse_qs, urlsplit import pytest -from litellm._redis_credential_provider import ( - ElastiCacheIAMCredentialProvider, - _BotocoreCredentials, -) +from litellm._redis_credential_provider import ElastiCacheIAMCredentialProvider class _FakeCredentials: @@ -20,16 +17,6 @@ class _FakeCredentials: return self -class _FakeResolver: - def __init__(self, credentials: _BotocoreCredentials | None) -> None: - self.credentials = credentials - self.calls = 0 - - def __call__(self): - self.calls += 1 - return self.credentials - - class _RotatingFakeCredentials: def __init__(self) -> None: self.calls = 0 @@ -46,6 +33,16 @@ class _RotatingFakeCredentials: ) +class _FakeResolver: + def __init__(self, credentials: _FakeCredentials | _RotatingFakeCredentials | None) -> None: + self.credentials = credentials + self.calls = 0 + + def __call__(self): + self.calls += 1 + return self.credentials + + def test_elasticache_provider_signs_expected_query(): resolver = _FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")) provider = ElastiCacheIAMCredentialProvider( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6d62ce2b675..9ca0df396b7 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26357,6 +26357,26 @@ export interface components { * independently of the response-cache backend in `litellm_settings.cache_params`. */ CoordinationRedisParams: { + /** + * Aws Iam Auth + * @description enable AWS ElastiCache IAM authentication + */ + aws_iam_auth?: boolean | string | null; + /** + * Aws Iam Cache Name + * @description AWS ElastiCache cache name + */ + aws_iam_cache_name?: string | null; + /** + * Aws Iam Region + * @description AWS region for ElastiCache IAM authentication + */ + aws_iam_region?: string | null; + /** + * Aws Iam User Name + * @description AWS ElastiCache IAM user name + */ + aws_iam_user_name?: string | null; /** * Host * @description Redis hostname From bb043fe29a7f680f8a5aee1ba5998f3fdb26e561 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 19:48:03 -0400 Subject: [PATCH 314/411] test(redis): cover ElastiCache IAM failures Generated with AI Co-Authored-By: Claude Code --- tests/test_litellm/test_redis.py | 52 ++++++-------- .../test_redis_credential_provider.py | 70 +++++++++++++++++++ 2 files changed, 93 insertions(+), 29 deletions(-) diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 21c23ba7866..43fb7563233 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -455,37 +455,33 @@ def test_explicit_provider_wins_over_aws_iam(clean_redis_environment): def test_gcp_wins_over_aws_iam(clean_redis_environment): - with patch("litellm._redis.create_gcp_iam_redis_connect_func") as mock_gcp: - mock_gcp.return_value = _gcp_marker_callback() - redis_kwargs = _get_redis_client_logic( - host="cache.example.com", - port=6379, - gcp_service_account="sa@example.com", - aws_iam_auth=True, - aws_iam_user_name="iam-user", - aws_iam_cache_name="cache.example.com", - aws_iam_region="us-east-1", - ) + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + gcp_service_account="sa@example.com", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) assert "credential_provider" not in redis_kwargs - assert redis_kwargs["redis_connect_func"] is mock_gcp.return_value + assert redis_kwargs["redis_connect_func"]._gcp_service_account == "sa@example.com" def test_azure_wins_over_aws_iam(clean_redis_environment): - with patch("litellm._redis.create_azure_ad_redis_connect_func") as mock_azure: - mock_azure.return_value = MagicMock() - redis_kwargs = _get_redis_client_logic( - host="cache.example.com", - port=6379, - azure_redis_ad_token="true", - aws_iam_auth=True, - aws_iam_user_name="iam-user", - aws_iam_cache_name="cache.example.com", - aws_iam_region="us-east-1", - ) + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + azure_redis_ad_token="true", + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) assert "credential_provider" not in redis_kwargs - assert redis_kwargs["redis_connect_func"] is mock_azure.return_value + assert redis_kwargs["redis_connect_func"]._azure_redis_ad_token is True def test_async_cluster_installs_aws_iam_provider(clean_redis_environment): @@ -1732,11 +1728,9 @@ def test_async_sentinel_keeps_the_credential_provider_off_the_monitors(markers, "redis_connect_func": SimpleNamespace(**markers), } - with ( - patch("litellm._redis.async_redis.Sentinel") as mock_sentinel_cls, - patch("litellm._redis._get_redis_client_logic", return_value=redis_kwargs), - ): - get_redis_async_client() + with patch("litellm._redis.async_redis.Sentinel") as mock_sentinel_cls: + with patch("litellm._redis._get_redis_client_logic", return_value=redis_kwargs): + get_redis_async_client() sentinel_kwargs = mock_sentinel_cls.call_args[1]["sentinel_kwargs"] assert sentinel_kwargs["password"] == sentinel_password diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index de4afb7c313..336e30e12c6 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -1,4 +1,6 @@ import asyncio +import builtins +import sys from types import SimpleNamespace from urllib.parse import parse_qs, urlsplit @@ -87,6 +89,41 @@ def test_elasticache_provider_resolves_credentials_once_but_refreshes_signature( assert rotating_credentials.calls == 3 +def test_elasticache_provider_uses_botocore_session_credentials(monkeypatch): + credentials = _FakeCredentials("AKIA-SYNTHETIC") + monkeypatch.setattr("botocore.session.get_session", lambda: SimpleNamespace(get_credentials=lambda: credentials)) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + ) + + user_name, token = provider.get_credentials() + + assert user_name == "iam-user" + assert "AKIA-SYNTHETIC" in token + + +def test_elasticache_provider_reports_missing_botocore(monkeypatch): + original_import = builtins.__import__ + + def import_without_botocore(name, *args, **kwargs): + if name == "botocore.session": + raise ImportError("synthetic missing dependency") + return original_import(name, *args, **kwargs) + + monkeypatch.delitem(sys.modules, "botocore.session", raising=False) + monkeypatch.setattr(builtins, "__import__", import_without_botocore) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + ) + + with pytest.raises(ImportError, match="pip install boto3"): + provider.get_credentials() + + def test_elasticache_provider_reports_missing_credentials(): provider = ElastiCacheIAMCredentialProvider( user_name="iam-user", @@ -99,6 +136,39 @@ def test_elasticache_provider_reports_missing_credentials(): provider.get_credentials() +def test_elasticache_provider_reports_missing_frozen_credentials(): + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=_FakeResolver(SimpleNamespace(get_frozen_credentials=lambda: None)), + ) + + with pytest.raises(RuntimeError, match="Unable to resolve AWS credentials"): + provider.get_credentials() + + +def test_elasticache_provider_reports_missing_signing_dependency(monkeypatch): + original_import = builtins.__import__ + + def import_without_botocore_auth(name, *args, **kwargs): + if name == "botocore.auth": + raise ImportError("synthetic missing dependency") + return original_import(name, *args, **kwargs) + + monkeypatch.delitem(sys.modules, "botocore.auth", raising=False) + monkeypatch.setattr(builtins, "__import__", import_without_botocore_auth) + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache.example.com", + region="us-east-1", + credentials_resolver=_FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")), + ) + + with pytest.raises(ImportError, match="pip install boto3"): + provider.get_credentials() + + def test_elasticache_provider_recovers_after_a_failed_resolution(): resolver = _FakeResolver(None) provider = ElastiCacheIAMCredentialProvider( From 921c263876600776be8dd69980f683b2c9ea3940 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 20:21:03 -0400 Subject: [PATCH 315/411] fix(proxy): validate coordination Redis mappings Generated with AI Co-Authored-By: Claude Code --- .../management_endpoints/coordination_redis_endpoints.py | 2 +- litellm/proxy/proxy_server.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py index 86ce336c7a3..88dc09ab001 100644 --- a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py +++ b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py @@ -140,7 +140,7 @@ def _merge_over_saved( def _validated_params(settings: Mapping[str, object]) -> CoordinationRedisParams: """Validate settings the way startup does: resolve env refs, then require a connection target.""" try: - params: Final = CoordinationRedisParams(**_resolve_env_refs(settings)) + params: Final = CoordinationRedisParams.model_validate(_resolve_env_refs(settings)) except ValidationError as e: invalid_fields: Final = sorted({str(error["loc"][0]) for error in e.errors() if error["loc"]}) raise HTTPException( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9269fd48e6c..c21df8ce931 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4906,7 +4906,7 @@ class ProxyConfig: if not isinstance(raw_params, dict): raise ValueError("general_settings.coordination_redis must be a mapping of Redis connection params") - coordination_params: Final = CoordinationRedisParams(**_resolve_coordination_redis_env_refs(raw_params)) + coordination_params: Final = CoordinationRedisParams.model_validate(_resolve_coordination_redis_env_refs(raw_params)) if not coordination_params.has_connection_target(): raise ValueError( "general_settings.coordination_redis needs a connection target: " @@ -9234,7 +9234,7 @@ class ProxyStartupEvent: if persisted is None: return None - coordination_params: Final = CoordinationRedisParams(**_resolve_coordination_redis_env_refs(persisted)) + coordination_params: Final = CoordinationRedisParams.model_validate(_resolve_coordination_redis_env_refs(persisted)) if not coordination_params.has_connection_target(): verbose_proxy_logger.warning( "coordination_redis saved in the database names no connection target; ignoring it." From 9763bd80ae442533d234e733cf334e3e1565681e Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 20:26:26 -0400 Subject: [PATCH 316/411] style(proxy): format coordination Redis validation Generated with AI Co-Authored-By: Claude Code --- litellm/proxy/proxy_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c21df8ce931..de10fde45c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4906,7 +4906,9 @@ class ProxyConfig: if not isinstance(raw_params, dict): raise ValueError("general_settings.coordination_redis must be a mapping of Redis connection params") - coordination_params: Final = CoordinationRedisParams.model_validate(_resolve_coordination_redis_env_refs(raw_params)) + coordination_params: Final = CoordinationRedisParams.model_validate( + _resolve_coordination_redis_env_refs(raw_params) + ) if not coordination_params.has_connection_target(): raise ValueError( "general_settings.coordination_redis needs a connection target: " @@ -9234,7 +9236,9 @@ class ProxyStartupEvent: if persisted is None: return None - coordination_params: Final = CoordinationRedisParams.model_validate(_resolve_coordination_redis_env_refs(persisted)) + coordination_params: Final = CoordinationRedisParams.model_validate( + _resolve_coordination_redis_env_refs(persisted) + ) if not coordination_params.has_connection_target(): verbose_proxy_logger.warning( "coordination_redis saved in the database names no connection target; ignoring it." From a7ad6023d6a52ed035b638c4b3ab867c27a3b65b Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 26 Aug 2026 21:14:07 -0400 Subject: [PATCH 317/411] ci: retry CodSpeed result upload Generated with AI Co-Authored-By: Claude Code From 0217a137fe3f8793163140fdb4d2f1fa4d132bbf Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Thu, 27 Aug 2026 13:49:12 -0400 Subject: [PATCH 318/411] fix(redis): require TLS for ElastiCache IAM auth --- litellm/_redis.py | 9 ++++ tests/test_litellm/test_redis.py | 78 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/litellm/_redis.py b/litellm/_redis.py index 2fd8a7100e3..50900b2977b 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -279,6 +279,13 @@ def _is_true(value: object | None) -> bool: return value is True or (isinstance(value, str) and value.lower() == "true") +def _uses_tls(redis_kwargs: Mapping[str, object]) -> bool: + if redis_kwargs.get("startup_nodes") is not None: + return _is_true(redis_kwargs.get("ssl")) + url: Final = redis_kwargs.get("url") + return urlsplit(url).scheme.lower() == "rediss" if isinstance(url, str) else _is_true(redis_kwargs.get("ssl")) + + def _build_elasticache_iam_provider( user_name: object | None, cache_name: object | None, @@ -604,6 +611,8 @@ def _get_redis_client_logic(**env_overrides): "for Redis. Using Azure AD. Remove one to avoid misconfiguration." ) elif _aws_iam_enabled: + if not _uses_tls(redis_kwargs): + raise ValueError("AWS ElastiCache IAM Redis authentication requires TLS") verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") redis_kwargs["credential_provider"] = _build_elasticache_iam_provider( user_name=redis_kwargs.get("aws_iam_user_name"), diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 43fb7563233..f2eda9a5753 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -319,6 +319,7 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, monkeypatch.setenv("REDIS_AWS_IAM_USER_NAME", "iam-user") monkeypatch.setenv("REDIS_AWS_IAM_CACHE_NAME", "cache.example.com") monkeypatch.setenv("REDIS_AWS_IAM_REGION", "us-east-1") + monkeypatch.setenv("REDIS_SSL", "true") redis_kwargs = _get_redis_client_logic(host="cache.example.com", port=6379) @@ -326,6 +327,77 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() +@pytest.mark.parametrize( + "transport", + [ + pytest.param({"host": "cache.example.com", "port": 6379}, id="host_without_ssl"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": False}, id="host_ssl_false"), + pytest.param({"url": "redis://cache.example.com:6379", "ssl": True}, id="plaintext_url"), + pytest.param( + {"startup_nodes": [{"host": "cache.example.com", "port": 6379}]}, + id="cluster_without_ssl", + ), + pytest.param( + { + "url": "rediss://cache.example.com:6379", + "startup_nodes": [{"host": "cache.example.com", "port": 6379}], + }, + id="cluster_without_ssl_ignores_url_scheme", + ), + pytest.param( + { + "sentinel_nodes": [("sentinel.example.com", 26379)], + "service_name": "cache", + }, + id="sentinel_without_ssl", + ), + ], +) +def test_aws_iam_auth_rejects_non_tls_connections(clean_redis_environment, transport): + with pytest.raises(ValueError, match="requires TLS"): + _get_redis_client_logic( + **transport, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + +@pytest.mark.parametrize( + "transport", + [ + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": True}, id="host"), + pytest.param({"url": "rediss://cache.example.com:6379"}, id="url"), + pytest.param( + { + "startup_nodes": [{"host": "cache.example.com", "port": 6379}], + "ssl": True, + }, + id="cluster", + ), + pytest.param( + { + "sentinel_nodes": [("sentinel.example.com", 26379)], + "service_name": "cache", + "ssl": True, + }, + id="sentinel", + ), + ], +) +def test_aws_iam_auth_accepts_tls_connections(clean_redis_environment, transport): + redis_kwargs = _get_redis_client_logic( + **transport, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache.example.com", + aws_iam_region="us-east-1", + ) + + assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + + def test_aws_iam_settings_are_removed_for_url_and_static_credentials(clean_redis_environment): redis_kwargs = _get_redis_client_logic( url="rediss://url-user:url-pass@cache.example.com:6380", @@ -351,6 +423,7 @@ def test_aws_iam_missing_setting_fails_closed(clean_redis_environment, missing): "aws_iam_user_name": "iam-user", "aws_iam_cache_name": "cache.example.com", "aws_iam_region": "us-east-1", + "ssl": True, } settings[missing] = None @@ -365,6 +438,7 @@ def test_aws_iam_region_falls_back_to_environment(clean_redis_environment, monke redis_kwargs = _get_redis_client_logic( host="cache.example.com", port=6379, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user", aws_iam_cache_name="cache.example.com", @@ -382,6 +456,7 @@ def test_aws_iam_region_prefers_aws_region_over_default_region(clean_redis_envir redis_kwargs = _get_redis_client_logic( host="cache.example.com", port=6379, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user", aws_iam_cache_name="cache.example.com", @@ -396,6 +471,7 @@ def test_aws_iam_region_prefers_explicit_over_environment(clean_redis_environmen redis_kwargs = _get_redis_client_logic( host="cache.example.com", port=6379, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user", aws_iam_cache_name="cache.example.com", @@ -409,6 +485,7 @@ def test_aws_iam_settings_map_to_distinct_provider_fields(clean_redis_environmen redis_kwargs = _get_redis_client_logic( host="cache.example.com", port=6379, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user-value", aws_iam_cache_name="iam-cache-value", @@ -489,6 +566,7 @@ def test_async_cluster_installs_aws_iam_provider(clean_redis_environment): client = get_redis_async_client( startup_nodes=startup_nodes, + ssl=True, aws_iam_auth=True, aws_iam_user_name="iam-user", aws_iam_cache_name="cache.example.com", From f1cd2b03caac2c544866be984089039aa1228956 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Mon, 31 Aug 2026 13:49:55 -0400 Subject: [PATCH 319/411] fix(redis): type signed ElastiCache IAM URL --- litellm/_redis_credential_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 85759af2a88..4d728ef21b6 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -4,7 +4,7 @@ import asyncio import threading import time from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Any, Final, Protocol, cast from urllib.parse import quote from redis.credentials import CredentialProvider @@ -185,7 +185,7 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): self._region, expires=self._token_lifetime_seconds, ).add_auth(request) - return self._user_name, request.url.removeprefix("https://") + return self._user_name, cast(str, request.url).removeprefix("https://") def get_credentials(self) -> tuple[str, str]: return self._get_credentials() From aca81c263ccd6660bffcf05115e4971eb92412c8 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Mon, 31 Aug 2026 14:07:04 -0400 Subject: [PATCH 320/411] fix(redis): validate signed ElastiCache IAM URL --- litellm/_redis_credential_provider.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 4d728ef21b6..4441d318373 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -4,7 +4,7 @@ import asyncio import threading import time from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing import TYPE_CHECKING, Any, Final, Protocol from urllib.parse import quote from redis.credentials import CredentialProvider @@ -185,7 +185,10 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): self._region, expires=self._token_lifetime_seconds, ).add_auth(request) - return self._user_name, cast(str, request.url).removeprefix("https://") + signed_url: Final = request.url + if signed_url is None: + raise RuntimeError("Unable to generate AWS ElastiCache IAM credentials") + return self._user_name, signed_url.removeprefix("https://") def get_credentials(self) -> tuple[str, str]: return self._get_credentials() From cd4073895ff9884d6fa77337fe6ac4c62dc029ec Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 10:03:26 -0400 Subject: [PATCH 321/411] refactor(redis): type botocore credentials without Any Deferred annotation evaluation keeps the type-checking-only botocore import off the runtime path, so the alias only reintroduced typing.Any, which the strict ruff budget now bans --- litellm/_redis_credential_provider.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 4441d318373..15f625dc8b4 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -4,15 +4,13 @@ import asyncio import threading import time from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Final, Protocol from urllib.parse import quote from redis.credentials import CredentialProvider if TYPE_CHECKING: from botocore.credentials import Credentials -else: - Credentials = Any # rebind-ok: runtime alias for the type-checking-only botocore import # Azure AD scope for Redis Cache for Azure. AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" From 2b0711e4f518dc796395972896aa5c3e6608b965 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 10:06:22 -0400 Subject: [PATCH 322/411] style(redis): restore the Azure AD marker comment The comment documents that the raw Azure client id, tenant id and secret are deliberately kept off the connect function, so this branch should never have dropped it --- litellm/_redis.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/litellm/_redis.py b/litellm/_redis.py index 50900b2977b..d77043a6dfd 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -598,6 +598,11 @@ def _get_redis_client_logic(**env_overrides): azure_tenant_id=_azure_tenant_id, azure_client_secret=_azure_client_secret, ) + # Marker for async paths to detect Azure AD auth. The live credential + # object is attached separately as `_azure_credential` by + # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret + # are intentionally NOT exposed on the function to avoid leaking + # credentials via inspection or logging. redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True if _aws_iam_enabled and _gcp_service_account is not None: From bf73c49d737b0ee1d4274a36a1846a8628de3478 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 10:17:53 -0400 Subject: [PATCH 323/411] refactor(redis): drop unreachable frozen credentials guard --- litellm/_redis_credential_provider.py | 2 -- tests/test_litellm/test_redis_credential_provider.py | 12 ------------ 2 files changed, 14 deletions(-) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 15f625dc8b4..dd49a6793ec 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -162,8 +162,6 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): self._credentials = credentials frozen_credentials: Final = credentials.get_frozen_credentials() - if frozen_credentials is None: - raise RuntimeError("Unable to resolve AWS credentials for ElastiCache IAM Redis authentication") try: from botocore.auth import SigV4QueryAuth diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index 336e30e12c6..9a58722b04e 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -136,18 +136,6 @@ def test_elasticache_provider_reports_missing_credentials(): provider.get_credentials() -def test_elasticache_provider_reports_missing_frozen_credentials(): - provider = ElastiCacheIAMCredentialProvider( - user_name="iam-user", - cache_name="cache.example.com", - region="us-east-1", - credentials_resolver=_FakeResolver(SimpleNamespace(get_frozen_credentials=lambda: None)), - ) - - with pytest.raises(RuntimeError, match="Unable to resolve AWS credentials"): - provider.get_credentials() - - def test_elasticache_provider_reports_missing_signing_dependency(monkeypatch): original_import = builtins.__import__ From c4fc20bcf933606d0ffc91a47036dc16df7059ea Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 14:43:58 -0400 Subject: [PATCH 324/411] fix(redis): accept every truthy flag and sign serverless ElastiCache caches The ElastiCache IAM gate read `aws_iam_auth` and `ssl` with a helper that only accepted the literal string "true", while the kwarg coercion that runs later accepts "true", "1" and "yes". Type coercion happens after the gate, so `REDIS_AWS_IAM_AUTH=1` silently skipped IAM auth and `REDIS_SSL=1` made the "requires TLS" check fail closed on a connection that was in fact TLS. Both helpers now share `_str_to_bool`. AWS signs serverless cache tokens with an extra `ResourceType=ServerlessCache` query parameter, so tokens minted for a serverless cache were rejected. Adds an `aws_iam_serverless` setting (`REDIS_AWS_IAM_SERVERLESS`) that puts the parameter into the signed URL, and lowercases the cache name because ElastiCache lowercases it at creation time. --- litellm/_redis.py | 51 ++++++------ litellm/_redis_credential_provider.py | 17 ++-- litellm/proxy/_types.py | 3 + .../cache_settings_endpoints.py | 9 ++ .../coordination_redis_endpoints.py | 8 ++ tests/test_litellm/test_redis.py | 82 +++++++++++++++++-- .../test_redis_credential_provider.py | 54 ++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 ++ 8 files changed, 192 insertions(+), 37 deletions(-) diff --git a/litellm/_redis.py b/litellm/_redis.py index d77043a6dfd..c5acdcb038b 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -39,6 +39,14 @@ from ._logging import verbose_logger AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" +_AWS_IAM_KWARG_NAMES: Final = ( + "aws_iam_auth", + "aws_iam_user_name", + "aws_iam_cache_name", + "aws_iam_region", + "aws_iam_serverless", +) + def _unwrapped_init_args(cls: type) -> frozenset[str]: """Every parameter on a single class's own ``__init__``, decorator-unwrapped. @@ -76,10 +84,7 @@ def _get_redis_kwargs(): "azure_client_id", "azure_tenant_id", "azure_client_secret", - "aws_iam_auth", - "aws_iam_user_name", - "aws_iam_cache_name", - "aws_iam_region", + *_AWS_IAM_KWARG_NAMES, } available_args: Final = {x for x in _unwrapped_init_args(redis.Redis) if x not in exclude_args} | include_args @@ -275,22 +280,25 @@ def _redis_kwargs_from_environment(): return return_dict -def _is_true(value: object | None) -> bool: - return value is True or (isinstance(value, str) and value.lower() == "true") +def _coerces_to_true(value: object | None) -> bool: + return _str_to_bool(value) if isinstance(value, str) else bool(value) def _uses_tls(redis_kwargs: Mapping[str, object]) -> bool: if redis_kwargs.get("startup_nodes") is not None: - return _is_true(redis_kwargs.get("ssl")) + return _coerces_to_true(redis_kwargs.get("ssl")) url: Final = redis_kwargs.get("url") - return urlsplit(url).scheme.lower() == "rediss" if isinstance(url, str) else _is_true(redis_kwargs.get("ssl")) + if isinstance(url, str): + return urlsplit(url).scheme.lower() == "rediss" + return _coerces_to_true(redis_kwargs.get("ssl")) -def _build_elasticache_iam_provider( - user_name: object | None, - cache_name: object | None, - region: object | None, -) -> ElastiCacheIAMCredentialProvider: +def _build_elasticache_iam_provider(redis_kwargs: Mapping[str, object]) -> ElastiCacheIAMCredentialProvider: + user_name: Final = redis_kwargs.get("aws_iam_user_name") + cache_name: Final = redis_kwargs.get("aws_iam_cache_name") + region: Final = ( + redis_kwargs.get("aws_iam_region") or get_secret_str("AWS_REGION") or get_secret_str("AWS_DEFAULT_REGION") + ) required_settings: Final = ( ("aws_iam_user_name", user_name), ("aws_iam_cache_name", cache_name), @@ -304,6 +312,7 @@ def _build_elasticache_iam_provider( user_name=str(user_name), cache_name=str(cache_name), region=str(region), + is_serverless=_coerces_to_true(redis_kwargs.get("aws_iam_serverless")), ) @@ -577,7 +586,7 @@ def _get_redis_client_logic(**env_overrides): _azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN") _azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true" - _aws_iam_enabled: Final = _is_true(redis_kwargs.get("aws_iam_auth")) + _aws_iam_enabled: Final = _coerces_to_true(redis_kwargs.get("aws_iam_auth")) if _azure_ad_enabled and _gcp_service_account is not None: verbose_logger.warning( @@ -619,13 +628,7 @@ def _get_redis_client_logic(**env_overrides): if not _uses_tls(redis_kwargs): raise ValueError("AWS ElastiCache IAM Redis authentication requires TLS") verbose_logger.debug("Setting up AWS ElastiCache IAM authentication for Redis.") - redis_kwargs["credential_provider"] = _build_elasticache_iam_provider( - user_name=redis_kwargs.get("aws_iam_user_name"), - cache_name=redis_kwargs.get("aws_iam_cache_name"), - region=redis_kwargs.get("aws_iam_region") - or get_secret_str("AWS_REGION") - or get_secret_str("AWS_DEFAULT_REGION"), - ) + redis_kwargs["credential_provider"] = _build_elasticache_iam_provider(redis_kwargs) redis_kwargs.pop("gcp_service_account", None) redis_kwargs.pop("gcp_ssl_ca_certs", None) @@ -635,10 +638,8 @@ def _get_redis_client_logic(**env_overrides): redis_kwargs.pop("azure_client_id", None) redis_kwargs.pop("azure_tenant_id", None) redis_kwargs.pop("azure_client_secret", None) - redis_kwargs.pop("aws_iam_auth", None) - redis_kwargs.pop("aws_iam_user_name", None) - redis_kwargs.pop("aws_iam_cache_name", None) - redis_kwargs.pop("aws_iam_region", None) + for aws_iam_key in _AWS_IAM_KWARG_NAMES: + redis_kwargs.pop(aws_iam_key, None) if redis_kwargs.get("credential_provider") is not None: redis_kwargs.pop("redis_connect_func", None) diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index dd49a6793ec..7d90f944657 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -5,7 +5,7 @@ import threading import time from collections.abc import Callable from typing import TYPE_CHECKING, Final, Protocol -from urllib.parse import quote +from urllib.parse import urlencode from redis.credentials import CredentialProvider @@ -126,6 +126,7 @@ class GCPIAMCredentialProvider(CredentialProvider): _ELASTICACHE_SERVICE_NAME: Final = "elasticache" _ELASTICACHE_TOKEN_TTL_SECONDS: Final = 900 +_ELASTICACHE_SERVERLESS_RESOURCE_TYPE: Final = "ServerlessCache" class ElastiCacheIAMCredentialProvider(CredentialProvider): @@ -134,12 +135,14 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): user_name: str, cache_name: str, region: str, + is_serverless: bool = False, credentials_resolver: Callable[[], Credentials | None] | None = None, token_lifetime_seconds: int = _ELASTICACHE_TOKEN_TTL_SECONDS, ) -> None: self._user_name = user_name - self._cache_name = cache_name + self._cache_name = cache_name.lower() self._region = region + self._is_serverless = is_serverless self._credentials_resolver = credentials_resolver or self._resolve_credentials self._credentials: Credentials | None = None self._token_lifetime_seconds = token_lifetime_seconds @@ -171,10 +174,14 @@ class ElastiCacheIAMCredentialProvider(CredentialProvider): "botocore is required for ElastiCache IAM Redis authentication. Install it with: pip install boto3" ) from e - request: Final = AWSRequest( - method="GET", - url=(f"https://{self._cache_name}/?Action=connect&User={quote(self._user_name, safe='')}"), + query: Final = urlencode( + ( + ("Action", "connect"), + ("User", self._user_name), + *((("ResourceType", _ELASTICACHE_SERVERLESS_RESOURCE_TYPE),) if self._is_serverless else ()), + ) ) + request: Final = AWSRequest(method="GET", url=f"https://{self._cache_name}/?{query}") SigV4QueryAuth( frozen_credentials, _ELASTICACHE_SERVICE_NAME, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 99103ff3706..07e4515059a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2431,6 +2431,9 @@ class CoordinationRedisParams(LiteLLMPydanticObjectBase): aws_iam_user_name: str | None = Field(None, description="AWS ElastiCache IAM user name") aws_iam_cache_name: str | None = Field(None, description="AWS ElastiCache cache name") aws_iam_region: str | None = Field(None, description="AWS region for ElastiCache IAM authentication") + aws_iam_serverless: bool | str | None = Field( + None, description="the ElastiCache cache is serverless rather than a self-designed cluster" + ) def has_connection_target(self) -> bool: return any(value is not None for value in (self.host, self.url, self.startup_nodes, self.sentinel_nodes)) diff --git a/litellm/types/management_endpoints/cache_settings_endpoints.py b/litellm/types/management_endpoints/cache_settings_endpoints.py index 10aece6efa4..cb05f3a50ac 100644 --- a/litellm/types/management_endpoints/cache_settings_endpoints.py +++ b/litellm/types/management_endpoints/cache_settings_endpoints.py @@ -273,4 +273,13 @@ CACHE_SETTINGS_FIELDS: Final[list[CacheSettingsField]] = [ ui_field_name="AWS IAM Region", redis_type=None, ), + CacheSettingsField( + field_name="aws_iam_serverless", + field_type="Boolean", + field_value=None, + field_description="The ElastiCache cache is serverless rather than a self-designed cluster", + field_default=False, + ui_field_name="AWS IAM Serverless Cache", + redis_type=None, + ), ] diff --git a/litellm/types/management_endpoints/coordination_redis_endpoints.py b/litellm/types/management_endpoints/coordination_redis_endpoints.py index 86b32752cbb..d70a921a22f 100644 --- a/litellm/types/management_endpoints/coordination_redis_endpoints.py +++ b/litellm/types/management_endpoints/coordination_redis_endpoints.py @@ -131,4 +131,12 @@ COORDINATION_REDIS_SETTINGS_FIELDS: Final[list[CoordinationRedisSettingsField]] ui_field_name="AWS IAM Region", section="connection", ), + CoordinationRedisSettingsField( + field_name="aws_iam_serverless", + field_type="Boolean", + field_description="The ElastiCache cache is serverless rather than a self-designed cluster", + field_default=False, + ui_field_name="AWS IAM Serverless Cache", + section="connection", + ), ] diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index f2eda9a5753..5337ed825a7 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -113,15 +113,25 @@ def test_credential_provider_is_not_environment_derived(): assert "credential_provider" not in mapping.values() +_AWS_IAM_SETTINGS = { + "aws_iam_auth", + "aws_iam_user_name", + "aws_iam_cache_name", + "aws_iam_region", + "aws_iam_serverless", +} + + def test_aws_iam_settings_are_environment_derived(): allowed = _get_redis_kwargs() mapping = _get_redis_env_kwarg_mapping() - assert {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} <= allowed + assert _AWS_IAM_SETTINGS <= allowed assert mapping["REDIS_AWS_IAM_AUTH"] == "aws_iam_auth" assert mapping["REDIS_AWS_IAM_USER_NAME"] == "aws_iam_user_name" assert mapping["REDIS_AWS_IAM_CACHE_NAME"] == "aws_iam_cache_name" assert mapping["REDIS_AWS_IAM_REGION"] == "aws_iam_region" + assert mapping["REDIS_AWS_IAM_SERVERLESS"] == "aws_iam_serverless" def test_sync_direct_preserves_credential_provider_identity(clean_redis_environment): @@ -319,12 +329,15 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, monkeypatch.setenv("REDIS_AWS_IAM_USER_NAME", "iam-user") monkeypatch.setenv("REDIS_AWS_IAM_CACHE_NAME", "cache.example.com") monkeypatch.setenv("REDIS_AWS_IAM_REGION", "us-east-1") - monkeypatch.setenv("REDIS_SSL", "true") + monkeypatch.setenv("REDIS_AWS_IAM_SERVERLESS", "1") + monkeypatch.setenv("REDIS_SSL", "1") redis_kwargs = _get_redis_client_logic(host="cache.example.com", port=6379) - assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) - assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + provider = redis_kwargs["credential_provider"] + assert isinstance(provider, ElastiCacheIAMCredentialProvider) + assert provider._is_serverless is True + assert not _AWS_IAM_SETTINGS & redis_kwargs.keys() @pytest.mark.parametrize( @@ -332,6 +345,9 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, [ pytest.param({"host": "cache.example.com", "port": 6379}, id="host_without_ssl"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": False}, id="host_ssl_false"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "false"}, id="host_ssl_false_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "0"}, id="host_ssl_zero_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "no"}, id="host_ssl_no_string"), pytest.param({"url": "redis://cache.example.com:6379", "ssl": True}, id="plaintext_url"), pytest.param( {"startup_nodes": [{"host": "cache.example.com", "port": 6379}]}, @@ -368,6 +384,13 @@ def test_aws_iam_auth_rejects_non_tls_connections(clean_redis_environment, trans "transport", [ pytest.param({"host": "cache.example.com", "port": 6379, "ssl": True}, id="host"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "true"}, id="host_ssl_true_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "1"}, id="host_ssl_one_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "yes"}, id="host_ssl_yes_string"), + pytest.param( + {"startup_nodes": [{"host": "cache.example.com", "port": 6379}], "ssl": "1"}, + id="cluster_ssl_one_string", + ), pytest.param({"url": "rediss://cache.example.com:6379"}, id="url"), pytest.param( { @@ -413,7 +436,7 @@ def test_aws_iam_settings_are_removed_for_url_and_static_credentials(clean_redis assert redis_kwargs["url"] == "rediss://cache.example.com:6380" assert "username" not in redis_kwargs assert "password" not in redis_kwargs - assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + assert not _AWS_IAM_SETTINGS & redis_kwargs.keys() @pytest.mark.parametrize("missing", ["aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"]) @@ -499,7 +522,7 @@ def test_aws_iam_settings_map_to_distinct_provider_fields(clean_redis_environmen assert provider._region == "iam-region-value" -@pytest.mark.parametrize("aws_iam_auth", [False, "false"]) +@pytest.mark.parametrize("aws_iam_auth", [None, False, "", "false", "0", "no"]) def test_aws_iam_auth_disabled_does_not_install_provider(clean_redis_environment, aws_iam_auth): redis_kwargs = _get_redis_client_logic( host="cache.example.com", @@ -511,7 +534,52 @@ def test_aws_iam_auth_disabled_does_not_install_provider(clean_redis_environment ) assert "credential_provider" not in redis_kwargs - assert not {"aws_iam_auth", "aws_iam_user_name", "aws_iam_cache_name", "aws_iam_region"} & redis_kwargs.keys() + assert not _AWS_IAM_SETTINGS & redis_kwargs.keys() + + +@pytest.mark.parametrize("aws_iam_auth", [True, "true", "True", "TRUE", "1", "yes"]) +def test_aws_iam_auth_enabled_by_any_truthy_flag(clean_redis_environment, aws_iam_auth): + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + ssl=True, + aws_iam_auth=aws_iam_auth, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache-name", + aws_iam_region="us-east-1", + ) + + assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) + + +@pytest.mark.parametrize( + "aws_iam_serverless, expected", + [ + pytest.param(None, False, id="unset"), + pytest.param(False, False, id="bool_false"), + pytest.param("false", False, id="string_false"), + pytest.param("0", False, id="string_zero"), + pytest.param(True, True, id="bool_true"), + pytest.param("true", True, id="string_true"), + pytest.param("1", True, id="string_one"), + ], +) +def test_aws_iam_serverless_flag_reaches_the_provider(clean_redis_environment, aws_iam_serverless, expected): + redis_kwargs = _get_redis_client_logic( + host="cache.example.com", + port=6379, + ssl=True, + aws_iam_auth=True, + aws_iam_user_name="iam-user", + aws_iam_cache_name="cache-name", + aws_iam_region="us-east-1", + aws_iam_serverless=aws_iam_serverless, + ) + + provider = redis_kwargs["credential_provider"] + assert isinstance(provider, ElastiCacheIAMCredentialProvider) + assert provider._is_serverless is expected + assert "aws_iam_serverless" not in redis_kwargs def test_explicit_provider_wins_over_aws_iam(clean_redis_environment): diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index 9a58722b04e..e71fc5d530d 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -175,3 +175,57 @@ def test_elasticache_provider_recovers_after_a_failed_resolution(): assert user_name == "iam-user" assert token assert resolver.calls == 2 + + +@pytest.mark.parametrize( + "provider_kwargs, expected_resource_type", + [ + pytest.param({}, None, id="default_is_self_designed"), + pytest.param({"is_serverless": False}, None, id="self_designed"), + pytest.param({"is_serverless": True}, ["ServerlessCache"], id="serverless"), + ], +) +def test_elasticache_provider_signs_resource_type_only_for_serverless(provider_kwargs, expected_resource_type): + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="cache-name", + region="us-east-1", + credentials_resolver=_FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")), + **provider_kwargs, + ) + + _, token = provider.get_credentials() + query = parse_qs(urlsplit("https://" + token).query) + + assert query.get("ResourceType") == expected_resource_type + assert query["X-Amz-Signature"] + + +def test_elasticache_provider_lowercases_the_cache_name(): + provider = ElastiCacheIAMCredentialProvider( + user_name="iam-user", + cache_name="Mixed-Case-Cache", + region="us-east-1", + credentials_resolver=_FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")), + ) + + _, token = provider.get_credentials() + + assert urlsplit("https://" + token).netloc == "mixed-case-cache" + + +def test_elasticache_provider_encodes_reserved_characters_in_the_user_name(): + user_name = "iam user/with+reserved&chars" + provider = ElastiCacheIAMCredentialProvider( + user_name=user_name, + cache_name="cache-name", + region="us-east-1", + credentials_resolver=_FakeResolver(_FakeCredentials("AKIA-SYNTHETIC")), + ) + + returned_user_name, token = provider.get_credentials() + query = parse_qs(urlsplit("https://" + token).query) + + assert returned_user_name == user_name + assert query["User"] == [user_name] + assert query["Action"] == ["connect"] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9ca0df396b7..0448f8373c1 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26372,6 +26372,11 @@ export interface components { * @description AWS region for ElastiCache IAM authentication */ aws_iam_region?: string | null; + /** + * Aws Iam Serverless + * @description the ElastiCache cache is serverless rather than a self-designed cluster + */ + aws_iam_serverless?: boolean | string | null; /** * Aws Iam User Name * @description AWS ElastiCache IAM user name From f0e3e031c3fd79acf04db3bca41c4f2f5d1f12ec Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Wed, 9 Sep 2026 15:01:06 -0400 Subject: [PATCH 325/411] test(redis): pin ElastiCache IAM signing and TLS coercion invariants Strengthens the serverless test to assert ResourceType is signed rather than merely present, ties _uses_tls to the redis-py kwarg coercion so the two cannot drift, locks the stripped-kwarg name tuple to the test's expectations, and adds "off" and "True" sentinel flag values. Renames the provider builder's parameter to redis_settings. --- tests/test_litellm/test_redis.py | 15 ++++++++++++++- .../test_redis_credential_provider.py | 19 ++++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 5337ed825a7..0e8c86d26df 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -10,13 +10,16 @@ from redis.credentials import CredentialProvider import litellm from litellm._redis import ( + _AWS_IAM_KWARG_NAMES, _async_auth_kwargs, + _coerce_redis_kwargs_types, _get_redis_client_logic, _get_redis_cluster_kwargs, _get_redis_env_kwarg_mapping, _get_redis_kwargs, _get_redis_url_kwargs, _pretty_print_redis_config, + _uses_tls, get_redis_async_client, get_redis_client, get_redis_connection_pool, @@ -31,6 +34,7 @@ from litellm._redis_credential_provider import ( from litellm.caching.redis_cache import RedisCache from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.constants import REDIS_CLUSTER_HEALTH_CHECK_INTERVAL +from litellm.proxy._types import CoordinationRedisParams class _StubCredentialProvider(CredentialProvider): @@ -127,6 +131,8 @@ def test_aws_iam_settings_are_environment_derived(): mapping = _get_redis_env_kwarg_mapping() assert _AWS_IAM_SETTINGS <= allowed + assert set(_AWS_IAM_KWARG_NAMES) == _AWS_IAM_SETTINGS + assert {f for f in CoordinationRedisParams.model_fields if f.startswith("aws_iam_")} == _AWS_IAM_SETTINGS assert mapping["REDIS_AWS_IAM_AUTH"] == "aws_iam_auth" assert mapping["REDIS_AWS_IAM_USER_NAME"] == "aws_iam_user_name" assert mapping["REDIS_AWS_IAM_CACHE_NAME"] == "aws_iam_cache_name" @@ -348,6 +354,7 @@ def test_aws_iam_environment_settings_install_provider(clean_redis_environment, pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "false"}, id="host_ssl_false_string"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "0"}, id="host_ssl_zero_string"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "no"}, id="host_ssl_no_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "off"}, id="host_ssl_off_string"), pytest.param({"url": "redis://cache.example.com:6379", "ssl": True}, id="plaintext_url"), pytest.param( {"startup_nodes": [{"host": "cache.example.com", "port": 6379}]}, @@ -385,6 +392,7 @@ def test_aws_iam_auth_rejects_non_tls_connections(clean_redis_environment, trans [ pytest.param({"host": "cache.example.com", "port": 6379, "ssl": True}, id="host"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "true"}, id="host_ssl_true_string"), + pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "True"}, id="host_ssl_true_capitalized"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "1"}, id="host_ssl_one_string"), pytest.param({"host": "cache.example.com", "port": 6379, "ssl": "yes"}, id="host_ssl_yes_string"), pytest.param( @@ -421,6 +429,11 @@ def test_aws_iam_auth_accepts_tls_connections(clean_redis_environment, transport assert isinstance(redis_kwargs["credential_provider"], ElastiCacheIAMCredentialProvider) +@pytest.mark.parametrize("ssl", ["true", "True", "TRUE", "1", "yes", "YES", "false", "0", "no", "off", "", "maybe"]) +def test_tls_detection_agrees_with_the_ssl_kwarg_coercion(ssl): + assert _uses_tls({"ssl": ssl}) is _coerce_redis_kwargs_types({"ssl": ssl})["ssl"] + + def test_aws_iam_settings_are_removed_for_url_and_static_credentials(clean_redis_environment): redis_kwargs = _get_redis_client_logic( url="rediss://url-user:url-pass@cache.example.com:6380", @@ -522,7 +535,7 @@ def test_aws_iam_settings_map_to_distinct_provider_fields(clean_redis_environmen assert provider._region == "iam-region-value" -@pytest.mark.parametrize("aws_iam_auth", [None, False, "", "false", "0", "no"]) +@pytest.mark.parametrize("aws_iam_auth", [None, False, "", "false", "0", "no", "off"]) def test_aws_iam_auth_disabled_does_not_install_provider(clean_redis_environment, aws_iam_auth): redis_kwargs = _get_redis_client_logic( host="cache.example.com", diff --git a/tests/test_litellm/test_redis_credential_provider.py b/tests/test_litellm/test_redis_credential_provider.py index e71fc5d530d..96b1933b0e3 100644 --- a/tests/test_litellm/test_redis_credential_provider.py +++ b/tests/test_litellm/test_redis_credential_provider.py @@ -178,14 +178,14 @@ def test_elasticache_provider_recovers_after_a_failed_resolution(): @pytest.mark.parametrize( - "provider_kwargs, expected_resource_type", + "provider_kwargs, expected_operation_params", [ - pytest.param({}, None, id="default_is_self_designed"), - pytest.param({"is_serverless": False}, None, id="self_designed"), - pytest.param({"is_serverless": True}, ["ServerlessCache"], id="serverless"), + pytest.param({}, frozenset({"Action", "User"}), id="default_is_self_designed"), + pytest.param({"is_serverless": False}, frozenset({"Action", "User"}), id="self_designed"), + pytest.param({"is_serverless": True}, frozenset({"Action", "User", "ResourceType"}), id="serverless"), ], ) -def test_elasticache_provider_signs_resource_type_only_for_serverless(provider_kwargs, expected_resource_type): +def test_elasticache_provider_signs_resource_type_only_for_serverless(provider_kwargs, expected_operation_params): provider = ElastiCacheIAMCredentialProvider( user_name="iam-user", cache_name="cache-name", @@ -195,9 +195,14 @@ def test_elasticache_provider_signs_resource_type_only_for_serverless(provider_k ) _, token = provider.get_credentials() - query = parse_qs(urlsplit("https://" + token).query) + query_string = urlsplit("https://" + token).query + param_names = tuple(pair.split("=", 1)[0] for pair in query_string.split("&")) + first_auth_param = next(i for i, name in enumerate(param_names) if name.startswith("X-Amz-")) + query = parse_qs(query_string) - assert query.get("ResourceType") == expected_resource_type + assert frozenset(param_names[:first_auth_param]) == expected_operation_params + assert all(name.startswith("X-Amz-") for name in param_names[first_auth_param:]) + assert query.get("ResourceType") == (["ServerlessCache"] if "ResourceType" in expected_operation_params else None) assert query["X-Amz-Signature"] From 2bf065f97d8b5cbc20fa1d57371b3c320baace38 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:09:18 -0700 Subject: [PATCH 326/411] fix(terraform): restore d.Partial(true) on a rejected /key/update (#40527) The squash of #40512 onto a base that already carried #40514 left the Partial call on the metadata pre-read error path only, so a rejected /key/update again persisted the planned values into state and TestResourceKeyUpdateFailureKeepsPriorState fails on the default branch. Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/provider/litellm/resource_key.go | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/provider/litellm/resource_key.go b/terraform/provider/litellm/resource_key.go index ffdc448f5fe..018d01f75a8 100644 --- a/terraform/provider/litellm/resource_key.go +++ b/terraform/provider/litellm/resource_key.go @@ -327,6 +327,7 @@ func resourceKeyUpdate(ctx context.Context, d *schema.ResourceData, m interface{ key.Metadata = metadata if _, err := c.UpdateKey(key); err != nil { + d.Partial(true) return diag.FromErr(fmt.Errorf("error updating key: %s", err)) } From f84034f5007ccf3be7e17a8bed401f12b94f51e8 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:22:26 -0700 Subject: [PATCH 327/411] feat(mock): report admission-time input token count in mock_response usage (#40590) * feat(mock): report admission-time input token count in mock_response usage Mock completions always reported prompt_tokens=10, so spend tracking, TPM metrics, budgets and the tokens-per-minute autoscaling signal saw 10 tokens for a 100k-token request. Budget reservation now carries the admission-time input token count in the reservation record, and mock_completion reads it back so mock traffic exercises the same spend and TPM paths as real traffic without any extra tokenizer work. * fix(mock): keep a zero admission input token count instead of falling back to 10 --------- Co-authored-by: yassin --- litellm/main.py | 34 +++++++++++++++++-- .../spend_tracking/budget_reservation.py | 1 + .../spend_tracking/test_budget_reservation.py | 11 ++++++ tests/test_litellm/test_main.py | 27 +++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 4f1c57adb53..1a4beb787bc 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -827,6 +827,29 @@ async def _sleep_for_timeout_async(timeout: float | str | httpx.Timeout): await asyncio.sleep(timeout.connect) +class _AdmissionReservation(BaseModel): + input_tokens: int | None = None + + +class _AdmissionMetadata(BaseModel): + user_api_key_budget_reservation: _AdmissionReservation | None = None + + +def admission_input_tokens(kwargs: Mapping[str, object]) -> int | None: + reservations: Final = ( + _AdmissionMetadata.model_validate(kwargs.get(key) or {}).user_api_key_budget_reservation + for key in ("litellm_metadata", "metadata") + ) + return next( + ( + reservation.input_tokens + for reservation in reservations + if reservation and reservation.input_tokens is not None + ), + None, + ) + + def mock_completion( model: str, messages: list, @@ -838,6 +861,7 @@ def mock_completion( logging=None, custom_llm_provider=None, timeout: float | str | httpx.Timeout | None = None, + prompt_tokens: int | None = None, **kwargs, ): """ @@ -953,13 +977,16 @@ def mock_completion( ChatCompletionMessageToolCall(**tool_call) for tool_call in mock_tool_calls ] + usage_prompt_tokens: Final = ( + prompt_tokens if prompt_tokens is not None else DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + ) setattr( model_response, "usage", Usage( - prompt_tokens=DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, + prompt_tokens=usage_prompt_tokens, completion_tokens=DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, - total_tokens=DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + total_tokens=usage_prompt_tokens + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, ), ) @@ -5550,6 +5577,9 @@ def completion( custom_llm_provider=custom_llm_provider, mock_timeout=mock_timeout, timeout=timeout, + prompt_tokens=admission_input_tokens( + cast(Mapping[str, object], kwargs) # cast-ok: completion's **kwargs is untyped + ), ) ## RESPONSES API BRIDGE LOGIC ## - check if model has 'mode: responses' in litellm.model_cost map diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index bf5eadcd85d..42f90957338 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -315,6 +315,7 @@ async def reserve_budget_for_request( "entries": applied_entries, "finalized": False, "input_cost": min(float(input_cost or 0.0), reservation_cost), + "input_tokens": max(input_token_counts.values(), default=None), } diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index de6c7c2a40a..31c51ecfd6b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -2,6 +2,7 @@ from typing import Final import pytest +import litellm import litellm.proxy.proxy_server as proxy_server from litellm.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth @@ -56,6 +57,16 @@ async def test_non_exempt_llm_route_still_reserves_budget(): assert reservation["reserved_cost"] > 0 +@pytest.mark.asyncio +async def test_reservation_carries_the_admission_input_token_count(): + reservation: Final = await _reserve("/v1/responses") + expected: Final = litellm.token_counter(model="gpt-4o", text="hello") + + assert reservation is not None + assert expected > 0 + assert reservation["input_tokens"] == expected + + ANTHROPIC_MESSAGES: Final = [{"role": "user", "content": "hello!!!"}] COUNT_TOKENS_REQUESTS: Final[tuple[tuple[str, dict[str, object]], ...]] = ( ("/v1/messages/count_tokens", {"model": "claude-sonnet-5", "messages": ANTHROPIC_MESSAGES}), diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 5cb52295f94..f8841af9750 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2394,6 +2394,33 @@ def test_image_edit_merges_headers_and_extra_headers(): assert "extra_headers" not in handler_kwargs["image_edit_optional_request_params"] +@pytest.mark.parametrize("metadata_key", ("metadata", "litellm_metadata")) +@pytest.mark.parametrize("input_tokens", (51234, 0)) +def test_mock_completion_usage_reports_admission_input_tokens(metadata_key: str, input_tokens: int): + response = litellm.completion( + model="anthropic/claude-sonnet-5", + messages=[{"role": "user", "content": "hello"}], + mock_response="ok", + api_key="mock", + **{metadata_key: {"user_api_key_budget_reservation": {"reserved_cost": 1.0, "input_tokens": input_tokens}}}, + ) + + assert response.usage.prompt_tokens == input_tokens + assert response.usage.total_tokens == input_tokens + response.usage.completion_tokens + + +def test_mock_completion_usage_falls_back_to_default_without_admission_count(): + response = litellm.completion( + model="anthropic/claude-sonnet-5", + messages=[{"role": "user", "content": "hello"}], + mock_response="ok", + api_key="mock", + metadata={"user_api_key_budget_reservation": {"reserved_cost": 1.0}}, + ) + + assert response.usage.prompt_tokens == litellm_main.DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + + def test_mock_completion_stream_with_model_response(): """Test that mock_completion correctly handles stream=True with a ModelResponse as mock_response.""" from litellm import completion From 6c69dd0f72e320ba249d2c46656d5d42e9b4119a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:22:46 -0700 Subject: [PATCH 328/411] perf(proxy): reuse cached model group and deployment info in budget reservation (#40593) Profiling the sidecar-enabled gateway at 700 rps showed ~2.4% of all samples in get_model_group_info called per request from budget reservation, plus get_deployment_model_info for tiered pricing tables. Both are read-only lookups over the model list, so serve them from the Router's lru caches and clear the deployment cache alongside the group cache when the model list changes. The deployment-info cache is a per-router lru_cache built in __init__ rather than a class-level decorated method, so it does not pin Router instances in a process-wide cache and is dropped with the router. A price data reload replaces litellm.model_cost without touching model_list, so the reload replay also clears both caches; otherwise reservation would keep pricing against the old catalog until an unrelated model-list change. Co-authored-by: yassin --- .../spend_tracking/budget_reservation.py | 4 +- litellm/router.py | 8 ++++ .../spend_tracking/test_budget_reservation.py | 47 +++++++++++++++++++ tests/test_litellm/test_router.py | 23 ++++++++- .../test_router_model_cost_isolation.py | 28 +++++++++++ 5 files changed, 107 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 42f90957338..8d4d72d9974 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1273,7 +1273,7 @@ def _get_model_cost_info( llm_router: Router | None, ) -> Mapping[str, object] | None: if llm_router is not None: - model_group_info: Final = llm_router.get_model_group_info(model_group=model) + model_group_info: Final = llm_router.cached_model_group_info(model) if model_group_info is not None: return model_group_info.model_dump() return dict(litellm.get_model_info(model=model)) @@ -1315,7 +1315,7 @@ def _deployment_tiered_pricing_table( backend_model: Final = _get_value(_get_value(deployment, "litellm_params"), "model") if not isinstance(model_id, str) or not isinstance(backend_model, str): return None - deployment_model_info: Final = llm_router.get_deployment_model_info(model_id=model_id, model_name=backend_model) + deployment_model_info: Final = llm_router.cached_deployment_model_info(model_id, backend_model) if deployment_model_info is None: return None tiered_pricing: Final = deployment_model_info.get("tiered_pricing") diff --git a/litellm/router.py b/litellm/router.py index a20f010c608..7ac8e889c2c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -943,6 +943,9 @@ class Router: # ``id()``-reuse risk after GC). See # ``litellm.proxy.auth.auth_checks._is_model_cost_zero``. self._zero_cost_cache: dict[str, bool] = {} + self.cached_deployment_model_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + self.get_deployment_model_info + ) self._routing_group_rows: tuple[DeploymentTypedDict, ...] | None = None self._init_routing_groups(None) self._provider_unresolved_deployments: tuple[Callable[[], Deployment | None], ...] = () @@ -10042,6 +10045,7 @@ class Router: model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, ) + self._invalidate_model_group_info_cache() def delete_deployment(self, id: str) -> Deployment | None: """ @@ -11003,6 +11007,9 @@ class Router: """ return self.get_model_group_info(model_group) + def cached_model_group_info(self, model_group: str) -> ModelGroupInfo | None: + return self._cached_get_model_group_info(model_group) + async def get_remaining_model_group_usage(self, model_group: str) -> dict[str, int]: model_group_info: Final = self._cached_get_model_group_info(model_group) @@ -11780,6 +11787,7 @@ class Router: result and bypass budget enforcement. """ self._cached_get_model_group_info.cache_clear() + self.cached_deployment_model_info.cache_clear() self._zero_cost_cache.clear() self._routing_group_rows = None diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index 31c51ecfd6b..e7dc5af9bfd 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -1,3 +1,4 @@ +import math from typing import Final import pytest @@ -9,6 +10,8 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.spend_tracking.budget_reservation import estimate_request_max_cost, reserve_budget_for_request from litellm.proxy.utils import ProxyLogging +from litellm.router import Router +from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo TOKEN_COUNTING_ROUTES: Final = ( "/responses/input_tokens", @@ -150,3 +153,47 @@ def test_bedrock_converse_body_reserves_the_prompt_not_the_context_window(): ) assert converse_cost is not None and invoke_cost is not None assert invoke_cost < converse_cost < 2 * invoke_cost + + +def _tiered_deployment(input_cost_per_token: float) -> Deployment: + return Deployment( + model_name="tiered-group", + litellm_params=LiteLLM_Params(model="dashscope/qwen3-max", api_key="sk-fake"), + model_info=ModelInfo( + id="tiered-deployment", + max_output_tokens=1000, + tiered_pricing=[ + { + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": input_cost_per_token, + "range": [0, 128000], + } + ], + ), + ) + + +TIERED_BODY: Final = {"model": "tiered-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10} + + +def test_repeated_estimates_reuse_cached_model_cost_info() -> None: + router: Final = Router(model_list=[_tiered_deployment(1e-06).model_dump()]) + first: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router) + hits_before: Final = router.cached_deployment_model_info.cache_info().hits + + second: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router) + + assert second == first + assert router.cached_deployment_model_info.cache_info().hits == hits_before + 1 + + +def test_deployment_pricing_update_invalidates_cached_estimate() -> None: + router: Final = Router(model_list=[_tiered_deployment(1e-06).model_dump()]) + before: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router) + assert before is not None + + router.upsert_deployment(_tiered_deployment(1e-03)) + + after: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router) + assert after is not None + assert math.isclose(after, before * 1000) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index d59d72a094e..bfa62698223 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -42,7 +42,7 @@ from litellm.router import ( _is_retriable_anthropic_status, ) from litellm.router_strategy import simple_shuffle -from litellm.types.router import DeploymentTypedDict, RetryPolicy +from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo, RetryPolicy def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata(): @@ -15140,3 +15140,24 @@ def test_deployment_ids_stringifies_ids_and_skips_entries_without_a_model_info_i {"no_model_info": True}, ) assert Router._deployment_ids(deployments) == frozenset({"a", "2"}) + + +def test_cached_model_info_lookups_match_uncached_and_reset_on_model_list_change(): + def deployment(max_output_tokens: int) -> Deployment: + return Deployment( + model_name="grp", + litellm_params=LiteLLM_Params(model="openai/gpt-4o", api_key="sk-a"), + model_info=ModelInfo(id="dep-a", max_output_tokens=max_output_tokens), + ) + + router = Router(model_list=[deployment(100).model_dump()]) + + assert router.cached_model_group_info("grp") == router.get_model_group_info("grp") + first = router.cached_deployment_model_info("dep-a", "openai/gpt-4o") + assert first == router.get_deployment_model_info(model_id="dep-a", model_name="openai/gpt-4o") + assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o") is first + + router.upsert_deployment(deployment(200)) + + assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["max_output_tokens"] == 200 + assert router.cached_model_group_info("grp").max_output_tokens == 200 diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 30b265905f3..bd38eecd1c6 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -2361,3 +2361,31 @@ def test_a_config_deployment_dropped_for_a_permanent_reason_is_not_retried_on_re assert router.get_model_names() == ["control-model"] assert router.deployment_names == names_after_boot + + +def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info(monkeypatch): + """ + Budget reservation reads pricing through the router's lru-cached group and + deployment lookups. A reload swaps the catalog without touching model_list, so + unless the replay clears those caches the next reservation prices against the + old catalog until some unrelated model-list change happens to evict it. + """ + router = Router( + model_list=[ + { + "model_name": "grp", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"}, + "model_info": {"id": "dep-a"}, + } + ] + ) + old_price = router.cached_model_group_info("grp").input_cost_per_token + assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == old_price + + new_price = old_price * 10 + fresh_catalog = copy.deepcopy(litellm.model_cost) + fresh_catalog["gpt-4o"]["input_cost_per_token"] = new_price + _simulate_price_data_reload_with_provider_sets(monkeypatch, fresh_catalog) + + assert router.cached_model_group_info("grp").input_cost_per_token == new_price + assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == new_price From 5d6fec94b74371a7314a4900f3cf025a54f8bcff Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 11:33:21 -0700 Subject: [PATCH 329/411] fix(router): strip Codex harness envelopes before classification --- .../complexity_router/README.md | 8 +- .../complexity_router/complexity_router.py | 9 +- .../complexity_router/config.py | 4 +- .../router_strategy/test_complexity_router.py | 103 +++++++++++++++++- 4 files changed, 118 insertions(+), 6 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index d605b43e42a..3d2b33a2ecd 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -446,7 +446,11 @@ Reasoning markers in the system prompt do **not** trigger the reasoning override Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead -By default a block is anything between `` and ``. `reminder_markers` replaces that with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: +By default the router strips complete ``, ``, ``, ``, and `` blocks. It also strips Codex repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path + +The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content + +`reminder_markers` replaces these defaults with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: ```yaml model_list: @@ -461,7 +465,7 @@ model_list: close: "[[SUBAGENT_CONTEXT_END]]" ``` -Setting `reminder_markers` replaces the built-in `` pair rather than adding to it, so list that pair too if your harness also emits it. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten +Setting `reminder_markers` replaces all built-in pairs, including the Codex heading pair, so include every default your harness still needs. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten ### Code Detection diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index faafcea404a..923aac85ee2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -386,7 +386,14 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | N _REMINDER_OPEN: Final = "" _REMINDER_CLOSE: Final = "" -_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),) +_DEFAULT_REMINDER_MARKERS: Final = ( + (_REMINDER_OPEN, _REMINDER_CLOSE), + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("# agents.md instructions for ", ""), +) _TRUNCATION_MARKER: Final = "..." _TRUNCATION_HEAD_FRACTION: Final = 0.3 diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index d28924c69b2..6ab4aa85148 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1292,8 +1292,8 @@ class ComplexityRouterConfig(BaseModel): "Override the delimiter pairs used to recognize and strip harness-injected reminder " "blocks before classification. A harness that wraps injected context differently per " "agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than " - "adds to, the built-in default of ('', ''), so a " - "harness that also emits that pair lists it too. Matching is case-insensitive." + "adds to, the built-in system-reminder and Codex envelope pairs, so list every " + "built-in pair your harness also emits. Matching is case-insensitive." ), ) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 51103297c58..792963fde23 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5,6 +5,7 @@ Tests the rule-based complexity scoring and tier assignment logic. """ import asyncio +from copy import deepcopy import logging import sys import time @@ -194,7 +195,14 @@ class TestComplexityRouterInit: complexity_router_config=basic_config, ) - assert router._reminder_markers == (("", ""),) + from litellm.router_strategy.complexity_router.complexity_router import _extract_current_ask_and_system_prompt + + assert ( + _extract_current_ask_and_system_prompt( + [{"role": "user", "content": "noisehello"}], router._reminder_markers + )[0] + == "hello" + ) def test_init_without_config(self, mock_router_instance): """Test initialization without configuration uses defaults.""" @@ -7601,11 +7609,103 @@ _ASKED = {"role": "user", "content": _ASK} _ANSWERED = {"role": "assistant", "content": "Working on it."} _TOOL_RESULT = {"type": "tool_result", "tool_use_id": "x", "content": "out"} _REMINDER = "Budget: 42 tokens remaining. Do not mention this." +_CODEX_NEW_TASK: Final = ( + "Message Type: NEW_TASK\nTask name: /root/cache_worker\nSender: /root\nPayload:\n" + "Implement and test a thread-safe bounded LRU cache." +) +_CODEX_ENVELOPES: Final = ( + "LITELLM ESCALATE cwd=/repo", + "LITELLM ESCALATE plugin list", + "LITELLM ESCALATE preferences", + "LITELLM ESCALATE environment", + "# AGENTS.md instructions for /repo with spaces/中文\nLITELLM ESCALATE instructions", +) class TestContextAwareClassifier: """Test the new classifier context window and trajectory signals.""" + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) + def test_codex_envelopes_preserve_delegated_task_and_prior_context(self, envelope: str) -> None: + from litellm.router_strategy.complexity_router.complexity_router import ( + _extract_current_ask_and_system_prompt, + _extract_prior_turns, + _newest_turn_ask, + _newest_turn_is_human_ask, + ) + + messages: Final = [ + {"role": "user", "content": f"{envelope}\nDesign cache invalidation"}, + { + "role": "user", + "content": [{"type": "text", "text": envelope}, {"type": "text", "text": _CODEX_NEW_TASK}], + }, + {"role": "developer", "content": "developer scope"}, + {"role": "user", "content": envelope}, + ] + + assert _extract_current_ask_and_system_prompt(messages)[0] == _CODEX_NEW_TASK + assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False) == ( + ("user", "Design cache invalidation"), + ) + assert _newest_turn_ask(messages) is None + assert _newest_turn_is_human_ask(messages) is False + assert _extract_current_ask_and_system_prompt([messages[-1]])[0] is None + + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) + def test_codex_marker_override_and_incomplete_blocks_preserve_text(self, envelope: str) -> None: + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + incomplete: Final = envelope.rsplit("noise{envelope}", (("", ""),)) == envelope + + @pytest.mark.asyncio + @pytest.mark.parametrize("responses_api", (False, True)) + async def test_codex_routing_preserves_original_request(self, responses_api: bool) -> None: + completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}')) + router: Final = ComplexityRouter( + model_name="codex-router", + litellm_router_instance=MagicMock(acompletion=completion), + complexity_router_config={ + "tiers": {"COMPLEX": "task-model", "REASONING": "escalated-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "keyword_tier_rules": [{"keywords": ["LITELLM ESCALATE"], "tier": "REASONING"}], + }, + ) + messages: Final = [ + {"role": "user", "content": _CODEX_NEW_TASK}, + {"role": "user", "content": "\n".join(_CODEX_ENVELOPES)}, + ] + original: Final = deepcopy(messages) + request_kwargs: Final = ( + {"input": messages, "litellm_metadata": {"user_api_key_request_route": "/v1/responses"}} + if responses_api + else {} + ) + + response: Final = await router.async_pre_routing_hook( + model="codex-router", + request_kwargs=request_kwargs, + messages=None if responses_api else messages, + input=messages if responses_api else None, + ) + + assert response is not None + assert response.model == "task-model" + completion.assert_awaited_once() + assert completion.call_args.kwargs["messages"][1]["content"].strip() == ( + f"Classify this message:\n{_CODEX_NEW_TASK}" + ) + assert messages == original + if responses_api: + assert response.messages is None + assert request_kwargs["input"] == original + else: + assert response.messages == original + @pytest.mark.parametrize( "messages,expected_ask", [ @@ -13662,6 +13762,7 @@ class _OutputCeilingRecorder(CustomLogger): def log_pre_api_call(self, model, messages, kwargs): self.seen.append((model, kwargs.get("optional_params", {}).get("max_tokens"))) + NON_REASONING_TIERS: Final = { "NON_REASONING": "gpt-4o-mini", "SIMPLE": "gpt-4o-mini", From 13ddd1ec6481756ac31cac4b7253e23eef3bf656 Mon Sep 17 00:00:00 2001 From: Wolfram Ravenwolf Date: Thu, 10 Sep 2026 20:47:13 +0200 Subject: [PATCH 330/411] fix(wandb): gate reasoning effort on model capabilities --- litellm/llms/wandb/chat/transformation.py | 6 +- ...odel_prices_and_context_window_backup.json | 40 +++++ model_prices_and_context_window.json | 40 +++++ .../wandb/test_wandb_chat_transformation.py | 170 ++++++++++++++---- 4 files changed, 219 insertions(+), 37 deletions(-) diff --git a/litellm/llms/wandb/chat/transformation.py b/litellm/llms/wandb/chat/transformation.py index 9c3ff818331..fdd6644f03d 100644 --- a/litellm/llms/wandb/chat/transformation.py +++ b/litellm/llms/wandb/chat/transformation.py @@ -6,12 +6,16 @@ This is OpenAI compatible - no translation needed / occurs from typing import Final +import litellm from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig class WandbConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract - return super().get_supported_openai_params(model) + ["reasoning_effort"] # mutable-ok: inherited contract + supported_params: Final = super().get_supported_openai_params(model) + if litellm.supports_reasoning(model=model, custom_llm_provider="wandb"): + return supported_params + ["reasoning_effort"] # mutable-ok: inherited contract + return supported_params def map_openai_params( self, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 55618d9f772..da47ef85eb1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45960,6 +45960,7 @@ "output_cost_per_token": 0.0 }, "wandb/openai/gpt-oss-120b": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -45970,6 +45971,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/openai/gpt-oss-20b": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -45980,6 +45982,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-4.5": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -46008,6 +46011,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -46064,6 +46068,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V3.1": { + "supports_reasoning": true, "max_tokens": 128000, "max_input_tokens": 161000, "max_output_tokens": 128000, @@ -46074,6 +46079,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-R1-0528": { + "supports_reasoning": true, "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, @@ -55631,6 +55637,7 @@ "supports_vision": false }, "wandb/deepseek-ai/DeepSeek-V4-Flash": { + "supports_reasoning": true, "max_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.4e-07, @@ -55643,6 +55650,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.3e-07, @@ -55655,6 +55663,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V4-Pro": { + "supports_reasoning": true, "max_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.15e-06, @@ -55667,6 +55676,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/google/gemma-4-31B-it": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, @@ -55707,6 +55717,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/MiniMaxAI/MiniMax-M3": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.3e-07, @@ -55719,6 +55730,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.7-Code": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.1e-07, @@ -55731,6 +55743,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.6": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6.5e-07, @@ -55743,6 +55756,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, @@ -55755,6 +55769,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.5e-07, @@ -55777,6 +55792,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.8-27B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, @@ -55789,6 +55805,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.6-35B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -55799,6 +55816,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.6-27B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6e-07, @@ -55811,6 +55829,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.5-35B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -55830,7 +55849,28 @@ "supports_vision": false, "source": "https://wandb.ai/site/pricing/tokens/" }, + "wandb/deepseek-ai/DeepSeek-V4-Pro-0813": { + "litellm_provider": "wandb", + "mode": "chat", + "supports_reasoning": true, + "input_cost_per_token": 0.00000131, + "output_cost_per_token": 0.00000396, + "cache_read_input_token_cost": 0.000000044, + "supports_prompt_caching": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/ibm-granite/granite-4.2-8b": { + "litellm_provider": "wandb", + "mode": "chat", + "supports_reasoning": true, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.00000015, + "cache_read_input_token_cost": 0.00000005, + "supports_prompt_caching": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, "wandb/zai-org/GLM-5.2": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.6e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 55618d9f772..da47ef85eb1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45960,6 +45960,7 @@ "output_cost_per_token": 0.0 }, "wandb/openai/gpt-oss-120b": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -45970,6 +45971,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/openai/gpt-oss-20b": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -45980,6 +45982,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/zai-org/GLM-4.5": { + "supports_reasoning": true, "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -46008,6 +46011,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -46064,6 +46068,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V3.1": { + "supports_reasoning": true, "max_tokens": 128000, "max_input_tokens": 161000, "max_output_tokens": 128000, @@ -46074,6 +46079,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-R1-0528": { + "supports_reasoning": true, "max_tokens": 161000, "max_input_tokens": 161000, "max_output_tokens": 161000, @@ -55631,6 +55637,7 @@ "supports_vision": false }, "wandb/deepseek-ai/DeepSeek-V4-Flash": { + "supports_reasoning": true, "max_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.4e-07, @@ -55643,6 +55650,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.3e-07, @@ -55655,6 +55663,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/deepseek-ai/DeepSeek-V4-Pro": { + "supports_reasoning": true, "max_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.15e-06, @@ -55667,6 +55676,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/google/gemma-4-31B-it": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, @@ -55707,6 +55717,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/MiniMaxAI/MiniMax-M3": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.3e-07, @@ -55719,6 +55730,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.7-Code": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.1e-07, @@ -55731,6 +55743,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.6": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6.5e-07, @@ -55743,6 +55756,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, @@ -55755,6 +55769,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.5e-07, @@ -55777,6 +55792,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.8-27B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, @@ -55789,6 +55805,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.6-35B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -55799,6 +55816,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.6-27B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6e-07, @@ -55811,6 +55829,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/Qwen/Qwen3.5-35B-A3B": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -55830,7 +55849,28 @@ "supports_vision": false, "source": "https://wandb.ai/site/pricing/tokens/" }, + "wandb/deepseek-ai/DeepSeek-V4-Pro-0813": { + "litellm_provider": "wandb", + "mode": "chat", + "supports_reasoning": true, + "input_cost_per_token": 0.00000131, + "output_cost_per_token": 0.00000396, + "cache_read_input_token_cost": 0.000000044, + "supports_prompt_caching": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, + "wandb/ibm-granite/granite-4.2-8b": { + "litellm_provider": "wandb", + "mode": "chat", + "supports_reasoning": true, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.00000015, + "cache_read_input_token_cost": 0.00000005, + "supports_prompt_caching": true, + "source": "https://wandb.ai/site/pricing/tokens/" + }, "wandb/zai-org/GLM-5.2": { + "supports_reasoning": true, "max_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.6e-07, diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index 4fd66448009..481a52bebad 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -6,30 +6,90 @@ Nebius AI Studio is an OpenAI-compatible provider with minor customizations. """ import json +from typing import Final import pytest +import respx import litellm from litellm import completion from litellm.llms.wandb.chat.transformation import WandbConfig +WANDB_REASONING_MODELS: Final = ( + "deepseek-ai/DeepSeek-V4-Flash", + "deepseek-ai/DeepSeek-V4-Flash-0731", + "deepseek-ai/DeepSeek-V4-Pro", + "deepseek-ai/DeepSeek-V4-Pro-0813", + "deepseek-ai/DeepSeek-V3.1", + "google/gemma-4-31B-it", + "ibm-granite/granite-4.2-8b", + "MiniMaxAI/MiniMax-M3", + "moonshotai/Kimi-K2.7-Code", + "moonshotai/Kimi-K2.6", + "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B", + "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", + "openai/gpt-oss-120b", + "openai/gpt-oss-20b", + "Qwen/Qwen3.8-27B", + "Qwen/Qwen3.6-35B-A3B", + "Qwen/Qwen3.6-27B", + "Qwen/Qwen3.5-35B-A3B", + "zai-org/GLM-5.2", + "moonshotai/Kimi-K2.5", + "MiniMaxAI/MiniMax-M2.5", + "zai-org/GLM-4.5", + "Qwen/Qwen3-235B-A22B-Thinking-2507", + "deepseek-ai/DeepSeek-R1-0528", +) + + +@pytest.fixture +def wandb_test_config(local_model_cost_map, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "telemetry", False) + monkeypatch.setattr(litellm, "drop_params", False) + + +@pytest.fixture +def wandb_request_mock(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post("https://api.inference.wandb.ai/v1/chat/completions").respond( + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Done"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + status_code=200, + ) + + class TestWandbConfig: """Test class for WandB Inference functionality""" - def test_map_openai_params_preserves_reasoning_effort(self): - supported_params = litellm.get_supported_openai_params(model="wandb/openai/gpt-oss-20b") + @pytest.mark.parametrize("model", WANDB_REASONING_MODELS) + def test_map_openai_params_preserves_reasoning_effort(self, wandb_test_config, model: str): + assert litellm.model_cost[f"wandb/{model}"].get("supports_reasoning") is True + supported_params = litellm.get_supported_openai_params(model=f"wandb/{model}") assert supported_params is not None assert "reasoning_effort" in supported_params result = WandbConfig().map_openai_params( - non_default_params={"reasoning_effort": "medium"}, + non_default_params={"reasoning_effort": "medium", "max_completion_tokens": 64}, optional_params={}, - model="openai/gpt-oss-20b", + model=model, drop_params=True, ) - assert result == {"reasoning_effort": "medium"} + assert result == {"reasoning_effort": "medium", "max_tokens": 64} def test_default_api_base(self): """Test that default API base is used when none is provided""" @@ -155,40 +215,78 @@ class TestWandbConfig: assert "Hey from LiteLLM" in content @pytest.mark.respx() - def test_wandb_completion_preserves_reasoning_effort_with_drop_params(self, respx_mock, monkeypatch): - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - - api_base = "https://api.inference.wandb.ai/v1" - request_mock = respx_mock.post(f"{api_base}/chat/completions").respond( - json={ - "id": "chatcmpl-123", - "object": "chat.completion", - "created": 1677652288, - "model": "gpt-oss-20b", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Done"}, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 1, - "completion_tokens": 1, - "total_tokens": 2, - }, - }, - status_code=200, - ) - + @pytest.mark.parametrize( + "model,effort", + tuple((model, "medium") for model in WANDB_REASONING_MODELS) + + ( + ("Qwen/Qwen3.8-27B", "low"), + ("Qwen/Qwen3.8-27B", "xhigh"), + ), + ) + def test_wandb_completion_preserves_reasoning_effort_with_drop_params( + self, wandb_test_config, wandb_request_mock: respx.Route, model: str, effort: str + ): completion( - model="wandb/openai/gpt-oss-20b", + model=f"wandb/{model}", messages=[{"role": "user", "content": "Hello"}], api_key="fake-wandb-key", - api_base=api_base, - reasoning_effort="medium", + api_base="https://api.inference.wandb.ai/v1", + reasoning_effort=effort, + max_completion_tokens=64, drop_params=True, ) - request_body = json.loads(request_mock.calls[0].request.content) - assert request_body["reasoning_effort"] == "medium" + assert wandb_request_mock.call_count == 1 + request_body = json.loads(wandb_request_mock.calls[0].request.content) + assert request_body["model"] == model + assert request_body["reasoning_effort"] == effort + assert request_body["max_tokens"] == 64 + assert "max_completion_tokens" not in request_body + + @pytest.mark.respx(assert_all_called=False) + @pytest.mark.parametrize("drop_params", [True, False]) + @pytest.mark.parametrize( + "model,explicit_false", + [ + ("meta-llama/Llama-3.1-8B-Instruct", False), + ("unknown-model", False), + ("openai/gpt-oss-20b", True), + ], + ) + def test_wandb_completion_without_reasoning_support( + self, + wandb_test_config, + wandb_request_mock: respx.Route, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model: str, + explicit_false: bool, + drop_params: bool, + ): + with monkeypatch.context() as context: + if explicit_false: + context.setitem(litellm.model_cost[f"wandb/{model}"], "supports_reasoning", False) + + kwargs = { + "model": f"wandb/{model}", + "messages": [{"role": "user", "content": "Hello"}], + "api_key": "fake-wandb-key", + "api_base": "https://api.inference.wandb.ai/v1", + "reasoning_effort": "medium", + "drop_params": drop_params, + } + if not drop_params: + with pytest.raises(litellm.UnsupportedParamsError, match="reasoning_effort"): + completion(**kwargs) + assert len(respx_mock.calls) == 0 + return + + completion(**kwargs) + assert wandb_request_mock.call_count == 1 + request_body = json.loads(wandb_request_mock.calls[0].request.content) + assert request_body["model"] == model + assert "reasoning_effort" not in request_body + + supported_params = litellm.get_supported_openai_params(model=f"wandb/{model}") + assert supported_params is not None + assert "reasoning_effort" not in supported_params From 153c962e15aa4a9506abc8dd7989b9d05f864faf Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 11:49:57 -0700 Subject: [PATCH 331/411] feat(router): log classifier input and masked source request --- litellm/integrations/custom_logger.py | 11 ++ .../litellm_core_utils/classifier_logging.py | 59 ++++++++ litellm/litellm_core_utils/litellm_logging.py | 27 +++- litellm/litellm_core_utils/redact_messages.py | 10 ++ litellm/llms/openai/openai.py | 2 + .../spend_management_endpoints.py | 20 ++- .../spend_tracking/spend_tracking_utils.py | 5 + .../complexity_router/complexity_router.py | 11 +- litellm/types/utils.py | 8 +- tests/e2e/coverage_registry/reliability.yaml | 1 + tests/e2e/router/test_classifier_audit_e2e.py | 143 ++++++++++++++++++ .../test_classifier_logging.py | 38 +++++ .../test_litellm_logging.py | 121 ++++++++++++++- .../test_redact_messages.py | 24 +++ .../test_spend_management_endpoints.py | 23 +++ .../test_spend_tracking_utils.py | 24 +++ .../router_strategy/test_complexity_router.py | 23 +++ .../ClassifierAuditView.test.tsx | 56 +++++++ .../LogDetailsDrawer/ClassifierAuditView.tsx | 52 +++++++ .../LogDetailsDrawer/LogDetailContent.tsx | 53 +++---- 20 files changed, 673 insertions(+), 38 deletions(-) create mode 100644 litellm/litellm_core_utils/classifier_logging.py create mode 100644 tests/e2e/router/test_classifier_audit_e2e.py create mode 100644 tests/test_litellm/litellm_core_utils/test_classifier_logging.py create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 8f03e08f02d..473bf223105 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -927,6 +927,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac # Handle turn_off_message_logging - redact messages and responses (if not already excluded) if turn_off_message_logging: + from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit + + for field in CLASSIFIER_AUDIT_FIELDS: + standard_logging_object_copy.pop(field, None) + params: Final = model_call_details_copy.get("litellm_params") + request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None + if isinstance(params, dict) and isinstance(request, dict): + model_call_details_copy["litellm_params"] = { + **params, + "proxy_server_request": without_classifier_audit(request), + } redacted_str: Final = "redacted-by-litellm" if "messages" not in (excluded_fields or []) and standard_logging_object_copy.get("messages") is not None: diff --git a/litellm/litellm_core_utils/classifier_logging.py b/litellm/litellm_core_utils/classifier_logging.py new file mode 100644 index 00000000000..b9cb52a190d --- /dev/null +++ b/litellm/litellm_core_utils/classifier_logging.py @@ -0,0 +1,59 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue, TypeAdapter, ValidationError + +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ClassifierAudit + +CLASSIFIER_AUDIT_FIELDS: Final = ("classifier_input", "originating_request_masked") +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def classifier_input_snapshot(value: object, *, openai_sdk: bool = False) -> Mapping[str, JsonValue] | None: + if openai_sdk and isinstance(value, Mapping): + body: Final = { + key: item for key, item in value.items() if key not in ("extra_headers", "extra_query", "extra_body") + } + extra_body: Final = value.get("extra_body") + return classifier_input_snapshot({**body, **extra_body} if isinstance(extra_body, Mapping) else body) + try: + return ( + _JSON_OBJECT.validate_json(value) + if isinstance(value, (str, bytes)) + else _JSON_OBJECT.validate_python(value) + ) + except ValidationError: + return None + + +def is_classifier_call(call_type: str, params: Mapping[str, object]) -> bool: + return call_type in ("completion", "acompletion") and any( + isinstance(metadata := params.get(key), Mapping) + and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == AUTOROUTER_CLASSIFIER_CALL_ORIGIN + for key in ("metadata", "litellm_metadata") + ) + + +def masked_originating_request(request_kwargs: Mapping[str, object] | None) -> Mapping[str, JsonValue] | None: + request: Final = (request_kwargs or {}).get("proxy_server_request") + body: Final = request.get("body") if isinstance(request, Mapping) else None + if not isinstance(body, Mapping): + return None + serializable: Final = classifier_input_snapshot(safe_dumps(body)) + return classifier_input_snapshot(redact_credentials_in_payload(serializable)) if serializable is not None else None + + +def classifier_audit_fields(payload: Mapping[str, object]) -> ClassifierAudit: + classifier_input: Final = classifier_input_snapshot(payload.get("classifier_input")) + originating_request: Final = classifier_input_snapshot(payload.get("originating_request_masked")) + return { + **(ClassifierAudit(classifier_input=classifier_input) if classifier_input is not None else {}), + **(ClassifierAudit(originating_request_masked=originating_request) if originating_request is not None else {}), + } + + +def without_classifier_audit(payload: Mapping[str, object]) -> dict[str, object]: + return {key: value for key, value in payload.items() if key not in CLASSIFIER_AUDIT_FIELDS} diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cb9209be267..f85aa32829f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -17,7 +17,7 @@ from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast from httpx import Response -from pydantic import BaseModel +from pydantic import BaseModel, JsonValue import litellm from litellm import ( @@ -64,6 +64,11 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger +from litellm.litellm_core_utils.classifier_logging import ( + classifier_audit_fields, + classifier_input_snapshot, + is_classifier_call, +) from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.internal_call_metadata import ( @@ -89,6 +94,7 @@ from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, redact_streaming_responses_for_custom_logger, + should_redact_message_logging, ) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse @@ -473,6 +479,7 @@ class Logging(LiteLLMLoggingBaseClass): stream_options = None litellm_request_debug: bool = False streamed_anthropic_message_id: str | None = None + classifier_input: Mapping[str, JsonValue] | None = None def __init__( self, @@ -1211,6 +1218,14 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["api_key"] = api_key self.model_call_details["additional_args"] = additional_args self.model_call_details["log_event_type"] = "pre_api_call" + if is_classifier_call(self.call_type, self.model_call_details.get("litellm_params") or {}): + self.classifier_input = ( + None + if should_redact_message_logging(self.model_call_details) + else classifier_input_snapshot( + additional_args.get("complete_input_dict"), openai_sdk=additional_args.get("openai_sdk") is True + ) + ) if model: # if model name was changes pre-call, overwrite the initial model call name with the new one self.model_call_details["model"] = model self.model_call_details["litellm_params"]["api_base"] = self._get_masked_api_base( @@ -6293,6 +6308,16 @@ def get_standard_logging_object_payload( ) payload: Final[StandardLoggingPayload] = StandardLoggingPayload( + **( + classifier_audit_fields( + { + "classifier_input": logging_obj.classifier_input, + "originating_request_masked": proxy_server_request.get("originating_request_masked"), + } + ) + if is_classifier_call(call_type or "", litellm_params) and not should_redact_message_logging(kwargs) + else {} + ), id=str(id), litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), trace_id=StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id( diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index a567854d767..c97566c21a8 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any, Final import litellm from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) @@ -183,6 +184,9 @@ def _redact_standard_logging_object(model_call_details: dict): redacted_str: Final = REDACTED_BY_LITELLM + for field in CLASSIFIER_AUDIT_FIELDS: + standard_logging_object.pop(field, None) + if standard_logging_object.get("messages") is not None: standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}] @@ -254,6 +258,12 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons copy via redact_streaming_responses_for_custom_logger instead. """ # Redact model_call_details + for field in CLASSIFIER_AUDIT_FIELDS: + model_call_details.pop(field, None) + params: Final = model_call_details.get("litellm_params") + request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None + if isinstance(params, dict) and isinstance(request, Mapping): + model_call_details["litellm_params"] = {**params, "proxy_server_request": without_classifier_audit(request)} model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] model_call_details["prompt"] = "" model_call_details["input"] = "" diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index edc8d64d9c2..5f04ebe0c01 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -797,6 +797,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): "api_base": openai_client._base_url._uri_reference, "acompletion": acompletion, "complete_input_dict": data, + "openai_sdk": True, }, ) @@ -938,6 +939,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): "api_base": openai_aclient._base_url._uri_reference, "acompletion": True, "complete_input_dict": data, + "openai_sdk": True, }, ) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index f8831ca4152..ee43972309b 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -26,7 +26,8 @@ from typing_extensions import ReadOnly import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME +from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME +from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, classifier_input_snapshot from litellm.proxy._types import * from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -3099,7 +3100,11 @@ async def _resolve_request_response_payload( proxy_server_request: Final = row.get("proxy_server_request") pg_payload: Final = RequestResponsePayload(messages, response, proxy_server_request) - if ( + stored_request: Final = classifier_input_snapshot(proxy_server_request) + truncated_audit: Final = bool(stored_request and classifier_audit_fields(stored_request)) and ( + LITELLM_TRUNCATED_PAYLOAD_FIELD in str(proxy_server_request) + ) + if not truncated_audit and ( _spend_log_field_has_content(messages) or _spend_log_field_has_content(response) or _spend_log_field_has_content(proxy_server_request) @@ -3124,10 +3129,19 @@ async def _resolve_request_response_payload( if payload is None: return pg_payload + cold_audit: Final = classifier_audit_fields(payload) + resolved_request: Final = ( + {**(classifier_input_snapshot(payload.get("proxy_server_request")) or stored_request or {}), **cold_audit} + if cold_audit + else payload.get("proxy_server_request") + ) + if truncated_audit: + return RequestResponsePayload(messages, response, resolved_request if cold_audit else proxy_server_request) + return RequestResponsePayload( messages=payload.get("messages"), response=payload.get("response"), - proxy_server_request=payload.get("proxy_server_request"), + proxy_server_request=resolved_request, ) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index a21d761996f..da55d1cb153 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -22,6 +22,7 @@ from litellm.constants import ( from litellm.constants import ( MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, ) +from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, @@ -1251,6 +1252,10 @@ def _get_proxy_server_request_for_spend_logs_payload( if _proxy_server_request is not None: _request_body = _proxy_server_request.get("body", {}) or {} + standard_payload: Final = (kwargs or {}).get("standard_logging_object") + if isinstance(standard_payload, Mapping): + _request_body = {**_request_body, **classifier_audit_fields(standard_payload)} + if kwargs is not None: realtime_tools: Final = kwargs.get("realtime_tools") if realtime_tools: diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index faafcea404a..162fcf2efb0 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -36,6 +36,7 @@ from litellm.constants import ( SESSION_ID_GENERATED_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.classifier_logging import masked_originating_request from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -1924,11 +1925,8 @@ class ComplexityRouter(CustomLogger): Call the configured classifier model with a system/user role split and prior-turn context. Builds a structured classification prompt with: - - System message: the stable classifier rubric AND the caller's own system prompt (task - constraints). This is the largest, most repeated part of the call, so keeping it in the - system role lets the provider prompt-cache it across a session's classifier calls. - - User message: the variable payload -- a few prior user turns for context and the current - ask to classify. + - System message: the stable classifier rubric. + - User message: the caller's system prompt quoted as task context, prior turns, and the current ask. Args: prompt: The current user ask text (already extracted as the real human ask, not tool results) @@ -2005,12 +2003,13 @@ class ComplexityRouter(CustomLogger): classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort}) proxy_server_request: Final = { + "originating_request_masked": masked_originating_request(request_kwargs), "body": { "model": llm_config.model, "messages": messages_for_call, "response_format": response_format, **classifier_call_params, - } + }, } classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ab0cc5f959c..58b940227f8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -35,6 +35,7 @@ from pydantic import ( BaseModel, ConfigDict, Field, + JsonValue, PrivateAttr, SkipValidation, field_serializer, @@ -3373,7 +3374,12 @@ class StandardAuditLogPayload(TypedDict): updated_values: str | None -class StandardLoggingPayload(TypedDict): +class ClassifierAudit(TypedDict, total=False): + classifier_input: ReadOnly[Mapping[str, JsonValue]] + originating_request_masked: ReadOnly[Mapping[str, JsonValue]] + + +class StandardLoggingPayload(ClassifierAudit): id: str trace_id: str # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries) session_id: str # End-user/conversation session id (litellm_session_id), independent of trace_id diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 6b69677d490..4d40f19e67b 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -1,4 +1,5 @@ # Reliability & Performance (behavior features). Grounded in litellm/router.py + router_strategy/ + router_utils/. +- {id: reliability.routing.classifier_audit.separates_provider_input_and_source, module: reliability, tier: P1, behavior: routing, variant: classifier_audit, assertions: [separates_provider_input_and_source], exercised_on: [chat_completions, messages, responses], source: "litellm/litellm_core_utils/classifier_logging.py", rationale: "Classifier spend details distinguish provider input from the credential-masked originating request across all three request surfaces"} - {id: reliability.fallback.5xx.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "5xx", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2024", rationale: "Reroute on provider 5xx to alternate deployment"} - {id: reliability.fallback.context_window.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: context_window, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6108", rationale: "Fallback when model exceeds context limit"} - {id: reliability.fallback.content_policy.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: content_policy, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6023", rationale: "Reroute on content-policy violation"} diff --git a/tests/e2e/router/test_classifier_audit_e2e.py b/tests/e2e/router/test_classifier_audit_e2e.py new file mode 100644 index 00000000000..6b444441477 --- /dev/null +++ b/tests/e2e/router/test_classifier_audit_e2e.py @@ -0,0 +1,143 @@ +import json +import os +from collections.abc import Iterator +from contextlib import ExitStack +from dataclasses import dataclass +from typing import Final, Literal + +import pytest +from pydantic import BaseModel, Field, JsonValue, TypeAdapter + +from e2e_config import unique_marker +from e2e_http import AnthropicHeaders, NoBody, unwrap +from models import ChatMessage, KeyGenerateBody, LiteLLMParamsBody +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + + +class AuditMetadata(BaseModel): + source_marker: str + authorization: str = "synthetic-audit-secret" + + +class AuditHeaders(AnthropicHeaders): + enable_redaction: str | None = Field(default=None, serialization_alias="x-litellm-enable-message-redaction") + + +class AuditRequest(BaseModel): + model: str + messages: list[ChatMessage] | None = None + system: str | None = None + instructions: str | None = None + input: str | None = None + max_tokens: int | None = None + max_output_tokens: int | None = None + metadata: AuditMetadata | None = None + litellm_metadata: AuditMetadata | None = None + + +class AuditResponse(BaseModel): + id: str + + +class AuditDetail(BaseModel): + proxy_server_request: dict[str, JsonValue] | str | None = None + response: dict[str, JsonValue] | str | None = None + + +@dataclass(frozen=True, slots=True) +class AuditDeployment: + alias: str + key: str + + +@pytest.fixture +def audit_deployment(proxy: ProxyClient, provider: str) -> Iterator[AuditDeployment]: + marker: Final = unique_marker() + classifier: Final = f"audit-classifier-{marker}" + target: Final = f"audit-target-{marker}" + alias: Final = f"audit-router-{marker}" + model: Final = os.environ.get( + f"E2E_CHEAP_{provider.upper()}_MODEL", "gpt-5.6" if provider == "openai" else "claude-haiku-4-5" + ) + params: Final = LiteLLMParamsBody( + model=f"{provider}/{model}", + api_key=os.environ.get(f"{provider.upper()}_API_KEY") or f"os.environ/{provider.upper()}_API_KEY", + api_base=os.environ.get(f"{provider.upper()}_API_BASE"), + ) + with ExitStack() as stack: + for name in (classifier, target): + stack.callback(proxy.delete_model, proxy.create_model(name, params)) + stack.callback(proxy.delete_model, proxy.create_model(alias, LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config={ + "classifier_type": "llm", + "classifier_llm_config": {"model": classifier, "timeout_ms": 30000}, + "tiers": {tier: target for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")}, + }, + ))) + key: Final = proxy.generate_key(KeyGenerateBody(models=[alias, classifier, target])) + stack.callback(proxy.delete_key, key) + yield AuditDeployment(alias, key) + + +class TestClassifierAudit: + @pytest.mark.covers( + "reliability.routing.classifier_audit.separates_provider_input_and_source", + exercised_on=("chat_completions", "messages", "responses"), + ) + @pytest.mark.parametrize("surface", ["chat_completions", "messages", "responses"]) + @pytest.mark.parametrize("provider", ["openai", "anthropic"]) + @pytest.mark.parametrize("redact", [False, True]) + def test_classifier_audit_separates_input_and_masked_source( + self, proxy: ProxyClient, audit_deployment: AuditDeployment, surface: Literal["chat_completions", "messages", "responses"], + redact: bool, + ) -> None: + marker: Final = unique_marker() + source_marker: Final = f"source-only-{marker}" + prompt: Final = f"Reply with hello. Request label {marker}" + metadata: Final = AuditMetadata(source_marker=source_marker) + body: Final = AuditRequest( + model=audit_deployment.alias, + messages=[ChatMessage(role="user", content=prompt)] if surface != "responses" else None, + input=prompt if surface == "responses" else None, + system="Be concise" if surface == "messages" else None, + instructions="Be concise" if surface == "responses" else None, + max_tokens=128 if surface != "responses" else None, + max_output_tokens=128 if surface == "responses" else None, + metadata=metadata if surface != "responses" else None, + litellm_metadata=metadata if surface == "responses" else None, + ) + path: Final = {"chat_completions": "/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}[surface] + response: Final = unwrap(proxy.transport.post( + path, headers=AuditHeaders(authorization=f"Bearer {audit_deployment.key}", enable_redaction="true" if redact else None), + json=body, response_type=AuditResponse, + )) + assert response.id + rows: Final = proxy.poll_logs_for_key(audit_deployment.key, min_rows=2) + assert len(rows) == 2, "Expected a classifier spend row and a routed response spend row" + details: Final = tuple( + unwrap(proxy.transport.get( + f"/spend/logs/ui/{row.request_id}", headers=proxy.transport.master, + params=NoBody(), response_type=AuditDetail, + )) for row in rows + ) + adapter: Final = TypeAdapter(dict[str, JsonValue]) + requests: Final = tuple( + adapter.validate_json(detail.proxy_server_request) if isinstance(detail.proxy_server_request, str) + else detail.proxy_server_request or {} for detail in details + ) + audits: Final = tuple(item for item in requests if "classifier_input" in item) + if redact: + assert audits == () + assert all("originating_request_masked" not in item for item in requests) + return + assert len(audits) == 1, "The audit belongs only to the classifier call" + audit: Final = audits[0] + assert marker in json.dumps(audit["classifier_input"]) + assert source_marker not in json.dumps(audit["classifier_input"]) + assert source_marker in json.dumps(audit["originating_request_masked"]) + assert "synthetic-audit-secret" not in json.dumps(audit) + assert audit_deployment.alias in json.dumps(audit["originating_request_masked"]) + assert any("tier" in str(detail.response) for detail in details) diff --git a/tests/test_litellm/litellm_core_utils/test_classifier_logging.py b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py new file mode 100644 index 00000000000..c923225c750 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py @@ -0,0 +1,38 @@ +from typing import Final + +import pytest + +from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot, masked_originating_request + + +@pytest.mark.parametrize("encoded", [False, True]) +def test_classifier_snapshot_preserves_provider_shape_and_is_independent(encoded: bool) -> None: + import json + + provider_body: Final = {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]} + snapshot: Final = classifier_input_snapshot(json.dumps(provider_body) if encoded else provider_body) + assert snapshot == provider_body + provider_body["messages"][0]["content"] = "later mutation" + assert snapshot == {"system": [{"text": "rubric"}], "messages": [{"role": "user", "content": "ask"}]} + + +def test_originating_snapshot_masks_nested_credentials_without_altering_source() -> None: + body: Final = { + "model": "router", + "input": [{"type": "message", "role": "user", "content": "source-only"}], + "api_key": "short", + "metadata": {"nested": [{"Authorization": "Bearer secret", "access_token": 123}]}, + } + snapshot: Final = masked_originating_request({"proxy_server_request": {"body": body}}) + assert snapshot is not None + assert snapshot["model"] == "router" + assert snapshot["input"] == body["input"] + assert snapshot["api_key"] == "REDACTED" + assert snapshot["metadata"] == {"nested": [{"Authorization": "REDACTED", "access_token": "REDACTED"}]} + assert body["api_key"] == "short" + assert body["metadata"]["nested"][0]["Authorization"] == "Bearer secret" + + +@pytest.mark.parametrize("value", [None, "not-json", [], {"messages": object()}]) +def test_invalid_provider_payload_is_not_reported_as_captured(value: object) -> None: + assert classifier_input_snapshot(value) is None diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 6aa77745e3d..28545bb2349 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3,7 +3,7 @@ import contextlib import datetime import os import sys -from typing import Literal +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -6801,3 +6801,122 @@ def test_get_error_information_redacts_provider_key_from_upstream_url(): assert "REDACTED" in result["traceback"] assert "REDACTED" in result["error_message"] assert result["error_code"] == "400" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ["openai", "anthropic", "bedrock"]) +async def test_classifier_audit_matches_provider_transport(provider: str) -> None: + import json + + from openai import AsyncOpenAI + + from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + outbound: Final = asyncio.Queue() + logs: Final = asyncio.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + outbound.put_nowait(json.loads(request.content)) + content: Final = '{"tier":"SIMPLE"}' + if provider == "anthropic": + return httpx.Response(200, json={ + "id": "msg-audit", "type": "message", "role": "assistant", "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": content}], "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }) + if provider == "bedrock": + return httpx.Response(200, json={ + "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, + "stopReason": "end_turn", "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + }) + return httpx.Response(200, json={ + "id": "chatcmpl-audit", "object": "chat.completion", "created": 0, "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }) + + async def capture(kwargs, response_obj, start_time, end_time): + logs.put_nowait(kwargs["standard_logging_object"]) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler: Final = AsyncHTTPHandler() + await handler.close() + handler.client = http_client + client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client) if provider == "openai" else handler + model: Final = { + "openai": "openai/gpt-5.6", + "anthropic": "anthropic/claude-haiku-4-5", + "bedrock": "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", + }[provider] + + async def run(marker: str) -> None: + await litellm.acompletion( + model=model, api_key="transport-only", client=client, max_tokens=128, + aws_access_key_id="transport-only", aws_secret_access_key="transport-only", aws_region_name="us-east-1", + messages=[{"role": "system", "content": "classifier-rubric"}, {"role": "user", "content": marker}], + metadata={"internal_call_origin": "autorouter_classifier"}, + proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, + success_callback=[capture], num_retries=0, + **({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}} + if provider == "openai" else {}), + ) + + await asyncio.gather(run("request-one"), run("request-two")) + requests: Final = await asyncio.wait_for(asyncio.gather(outbound.get(), outbound.get()), timeout=10) + payloads: Final = await asyncio.wait_for(asyncio.gather(logs.get(), logs.get()), timeout=10) + for payload in payloads: + snapshot: Final = payload["classifier_input"] + assert snapshot in requests + assert "source-only" not in json.dumps(snapshot) + assert "classifier-rubric" in json.dumps(snapshot) + assert "transport-only" not in json.dumps(snapshot) + assert "header-only-secret" not in json.dumps(snapshot) + assert "SIMPLE" in json.dumps(payload["response"]) + marker: Final = "request-one" if "request-one" in json.dumps(snapshot) else "request-two" + assert payload["originating_request_masked"] == {"input": f"source-only-{marker}"} + assert classifier_input_snapshot(snapshot) is not None + if provider != "openai": + assert all("system" in request for request in requests) + + +@pytest.mark.parametrize("redaction", ["none", "global", "request", "header"]) +@pytest.mark.parametrize("status", ["success", "failure"]) +def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_obj, monkeypatch, redaction, status): + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + monkeypatch.setattr(litellm, "turn_off_message_logging", redaction == "global") + params: Final = { + "metadata": {"internal_call_origin": "autorouter_classifier", **( + {"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {} + )}, + "proxy_server_request": {"body": {}, "originating_request_masked": {"input": "source-only"}}, + } + logging_obj.model_call_details["litellm_params"] = params + logging_obj.model_call_details["standard_callback_dynamic_params"] = ( + {"turn_off_message_logging": True} if redaction == "request" else {} + ) + logging_obj.pre_call( + input=[], api_key=None, additional_args={"complete_input_dict": {"system": "rubric", "messages": []}} + ) + now: Final = datetime.datetime.now() + payload: Final = get_standard_logging_object_payload( + kwargs={**logging_obj.model_call_details, "call_type": "completion"}, init_response_obj={}, + start_time=now, end_time=now, logging_obj=logging_obj, status=status, + ) + assert payload is not None + if redaction == "none": + assert payload["classifier_input"] == {"system": "rubric", "messages": []} + assert payload["originating_request_masked"] == {"input": "source-only"} + else: + assert "classifier_input" not in payload + assert "originating_request_masked" not in payload + + +@pytest.mark.parametrize("call_type,origin", [("completion", None), ("aembedding", "autorouter_classifier")]) +def test_classifier_audit_is_not_added_to_other_calls(logging_obj, call_type, origin): + logging_obj.call_type = call_type + logging_obj.model_call_details["litellm_params"] = {"metadata": {"internal_call_origin": origin}} + logging_obj.pre_call(input=[], api_key=None, additional_args={"complete_input_dict": {"input": "embedding"}}) + assert logging_obj.classifier_input is None diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 3be0bae4120..1ba9901aab0 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -6,6 +6,7 @@ but litellm_params["litellm_metadata"] is None. """ import threading +from typing import Final from types import SimpleNamespace import pytest @@ -776,6 +777,29 @@ class TestPerformRedaction: assert response_obj.choices[0].message.content == "secret content" + +@pytest.mark.parametrize("callback_only", [False, True]) +def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None: + audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}} + details: Final = { + "standard_logging_object": {**audit, "messages": [], "response": {}}, + "litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}}, + } + logger: Final = CustomLogger() + logger.turn_off_message_logging = True + if callback_only: + redacted: Final = logger.redact_standard_logging_payload_from_model_call_details(details) + assert "classifier_input" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"] + assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"] + assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"] + else: + perform_redaction(details, result=None) + assert "classifier_input" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["litellm_params"]["proxy_server_request"] + def test_unredactable_result_is_not_deepcopied(self): """A result shape no branch can redact must not be deepcopied. diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 329a33eb440..a65209b1456 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -5703,6 +5703,29 @@ def _cold_storage_handler(payload): return ColdStorageHandler(cold_storage_logger=logger), logger +@pytest.mark.asyncio +@pytest.mark.parametrize("cold_has_audit", [False, True]) +async def test_resolve_payload_recovers_truncated_classifier_audit_without_losing_existing_fields(cold_has_audit): + full_audit = {"classifier_input": {"system": "full rubric"}, "originating_request_masked": {"input": "source"}} + truncated_request = {"model": "classifier", "classifier_input": {"system": "litellm_truncated"}} + handler, logger = _cold_storage_handler({ + "proxy_server_request": {"body": {}}, **(full_audit if cold_has_audit else {}), + }) + row = { + "messages": '[{"role":"user","content":"ask"}]', "response": '{"tier":"SIMPLE"}', + "proxy_server_request": json.dumps(truncated_request), "metadata": {"cold_storage_object_key": "k/audit.json"}, + } + resolved = await spend_management_endpoints._resolve_request_response_payload(row, cold_storage_handler=handler) + assert logger.requested_object_keys == ["k/audit.json"] + assert resolved.messages == row["messages"] + assert resolved.response == row["response"] + if cold_has_audit: + assert resolved.proxy_server_request["classifier_input"] == full_audit["classifier_input"] + assert resolved.proxy_server_request["originating_request_masked"] == full_audit["originating_request_masked"] + else: + assert resolved.proxy_server_request == row["proxy_server_request"] + + @pytest.mark.parametrize( "value, expected", [ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 5a79560b972..adc15870968 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -67,6 +67,30 @@ def _get_additional_usage_values_for_usage(usage: litellm.Usage) -> dict: return metadata["additional_usage_values"] +@pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)]) +def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": store_prompts}) + audit: Final = { + "classifier_input": {"system": "rubric" * 1000, "messages": [{"role": "user", "content": "ask"}]}, + "originating_request_masked": {"input": "source-only", "api_key": "REDACTED"}, + } + stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload( + metadata={}, litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}}, + kwargs={"standard_logging_object": audit, "standard_callback_dynamic_params": {"turn_off_message_logging": redact}}, + )) + if not store_prompts or redact: + assert "classifier_input" not in stored + assert "originating_request_masked" not in stored + else: + assert stored["classifier_input"]["messages"] == audit["classifier_input"]["messages"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in json.dumps(stored["classifier_input"]) + assert stored["originating_request_masked"]["input"] == "source-only" + assert stored["model"] == "classifier" + assert audit["classifier_input"]["system"] == "rubric" * 1000 + + def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_tokens(): additional_usage_values = _get_additional_usage_values_for_usage( litellm.Usage( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 51103297c58..45b2faea126 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2940,6 +2940,29 @@ class TestLLMClassifier: "REASONING", ] + @pytest.mark.asyncio + @pytest.mark.parametrize("source_body", [ + {"model": "router", "messages": [{"role": "user", "content": "source-only"}]}, + {"model": "router", "system": "source-only", "messages": [{"role": "user", "content": "ask"}]}, + {"model": "router", "instructions": "source-only", "input": "ask"}, + ]) + async def test_classifier_source_is_masked_and_separate_from_provider_input( + self, llm_complexity_router, mock_router_instance, source_body + ): + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) + outcome = await llm_complexity_router.aclassify( + "classify-this-ask", request_kwargs={"proxy_server_request": { + "body": {**source_body, "metadata": {"authorization": "source-secret"}} + }} + ) + assert outcome.cause == "llm_classifier" + call_kwargs = mock_router_instance.acompletion.call_args.kwargs + source = call_kwargs["proxy_server_request"]["originating_request_masked"] + assert source == {**source_body, "metadata": {"authorization": "REDACTED"}} + assert "source-only" not in str(call_kwargs["messages"]) + assert "source-only" not in str(call_kwargs["proxy_server_request"]["body"]) + assert "classify-this-ask" in str(call_kwargs["messages"]) + @pytest.mark.asyncio @pytest.mark.parametrize("reasoning_effort", [None, "none", "low"], ids=["omitted", "none", "low"]) async def test_classifier_reasoning_effort_reaches_only_classifier_call( diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx new file mode 100644 index 00000000000..86e98de5557 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.test.tsx @@ -0,0 +1,56 @@ +import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { ClassifierAuditView } from "./ClassifierAuditView"; + +vi.mock("./JsonViewer", () => ({ + JsonViewer: ({ data }: { data: unknown }) => 
{JSON.stringify(data)}
, +})); + +describe("ClassifierAuditView", () => { + it("separates and copies the provider input, source request, and returned verdict", async () => { + const user = userEvent.setup(); + const input = { system: "classification rubric", messages: [{ role: "user", content: "classify this" }] }; + render( + , + ); + const classifier = within(screen.getByRole("region", { name: "Classifier input" })); + expect(classifier.getByText(/classification rubric/)).toBeInTheDocument(); + expect(classifier.queryByText(/source-only/)).not.toBeInTheDocument(); + expect( + within(screen.getByRole("region", { name: "Originating request, credentials masked" })).getByText(/source-only/), + ).toBeInTheDocument(); + expect( + within(screen.getByRole("region", { name: "Classifier response" })).getByText(/a greeting/), + ).toBeInTheDocument(); + await user.click(classifier.getByRole("button", { name: "Copy Classifier input" })); + expect(await navigator.clipboard.readText()).toBe(JSON.stringify(input, null, 2)); + }); + + it("does not present legacy source messages as captured classifier input", () => { + render(); + expect(screen.getAllByText("Not captured or message logging disabled")).toHaveLength(3); + expect(screen.queryByRole("button", { name: "Copy Classifier input" })).not.toBeInTheDocument(); + }); + + it("labels truncated input without marking a complete source request as truncated", () => { + render( + , + ); + expect(within(screen.getByRole("region", { name: "Classifier input" })).getByRole("status")).toHaveTextContent( + "This stored copy is truncated", + ); + expect( + within(screen.getByRole("region", { name: "Originating request, credentials masked" })).queryByRole("status"), + ).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx new file mode 100644 index 00000000000..913700efa9b --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ClassifierAuditView.tsx @@ -0,0 +1,52 @@ +import CopyButton from "@/components/shared/CopyButton"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { ReactNode } from "react"; +import { JsonViewer } from "./JsonViewer"; + +interface ClassifierAuditViewProps { + request: Record; + response: unknown; +} + +export function ClassifierAuditView({ request, response }: ClassifierAuditViewProps) { + return ( +
+ + Provider request payload. A cached call or disabled message logging may have no capture. + + + Comparison only. This source request was not appended to the classifier input. + + + The returned verdict and any explanation supplied by the classifier. Later routing rules may change the tier. + +
+ ); +} + +function AuditField({ title, value, children }: { title: string; value: unknown; children: ReactNode }) { + const serialized = JSON.stringify(value); + const truncated = serialized?.includes("litellm_truncated") ?? false; + + return ( + + + {title} + {value != null && } + + +

{children}

+ {truncated && ( +

+ This stored copy is truncated. The complete payload is unavailable from the configured log storage. +

+ )} + {value == null ? ( +

Not captured or message logging disabled

+ ) : ( + + )} + + + ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index 186cb66592b..e29f2810926 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -48,6 +48,8 @@ import { } from "./constants"; import { ToolsSection } from "../ToolsSection"; import { PrettyMessagesView } from "./PrettyMessagesView"; +import { ClassifierAuditView } from "./ClassifierAuditView"; +import { AUTOROUTER_CLASSIFIER_ORIGIN } from "./ClassifyTag"; export interface LogDetailContentProps { logEntry: LogEntry; @@ -68,6 +70,9 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok const metadata = logEntry.metadata || {}; const hasError = metadata.status === "failure"; const errorInfo = hasError ? metadata.error_information : null; + const isClassifier = + metadata.internal_call_origin === AUTOROUTER_CLASSIFIER_ORIGIN && + (logEntry.call_type === "completion" || logEntry.call_type === "acompletion"); const hasMessages = checkHasMessages(logEntry.messages); const hasResponse = checkHasResponse(logEntry.response); @@ -196,7 +201,11 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok Loading request & response data...
- ) : ( + ) : null} + {!isLoadingDetails && isClassifier && ( + + )} + {!isLoadingDetails && !isClassifier && ( - {metadata?.attempted_retries !== undefined && metadata?.attempted_retries !== null ? ( - metadata.attempted_retries > 0 ? ( - <> - {metadata.attempted_retries} - {metadata.max_retries !== undefined && metadata.max_retries !== null - ? ` / ${metadata.max_retries}` - : ""} - - ) : ( - - None - - ) - ) : ( - "-" + {metadata?.attempted_retries != null && metadata.attempted_retries > 0 && ( + <> + {metadata.attempted_retries} + {metadata.max_retries !== undefined && metadata.max_retries !== null + ? ` / ${metadata.max_retries}` + : ""} + )} + {metadata?.attempted_retries != null && metadata.attempted_retries <= 0 && ( + + None + + )} + {metadata?.attempted_retries == null && "-"} @@ -602,16 +609,10 @@ function RequestResponseSection({ const totalTokens = promptTokens + completionTokens; const costBreakdown = logEntry.metadata?.cost_breakdown; const useCostBreakdown = costBreakdown?.input_cost !== undefined && costBreakdown?.output_cost !== undefined; - const inputCost = useCostBreakdown - ? costBreakdown!.input_cost ?? 0 - : totalTokens > 0 - ? (totalSpend * promptTokens) / totalTokens - : 0; - const outputCost = useCostBreakdown - ? costBreakdown!.output_cost ?? 0 - : totalTokens > 0 - ? (totalSpend * completionTokens) / totalTokens - : 0; + const estimatedInputCost = totalTokens > 0 ? (totalSpend * promptTokens) / totalTokens : 0; + const estimatedOutputCost = totalTokens > 0 ? (totalSpend * completionTokens) / totalTokens : 0; + const inputCost = useCostBreakdown ? costBreakdown!.input_cost ?? 0 : estimatedInputCost; + const outputCost = useCostBreakdown ? costBreakdown!.output_cost ?? 0 : estimatedOutputCost; return (
From 1697684b685278ac68764a2f7278ffee9f05d2ff Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:01:04 -0700 Subject: [PATCH 332/411] fix(router): scope Codex envelope defaults to Codex clients --- litellm/litellm_core_utils/core_helpers.py | 8 ++ litellm/proxy/litellm_pre_call_utils.py | 15 +-- .../complexity_router/README.md | 4 +- .../complexity_router/complexity_router.py | 40 +++++--- .../complexity_router/config.py | 5 +- .../router_strategy/test_complexity_router.py | 96 +++++++++++++++++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 7 files changed, 127 insertions(+), 43 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index eacc3e4860a..aa7d6ca1699 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import re from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal @@ -21,6 +22,13 @@ else: Span = Any +_CODEX_CLIENT_PREFIX_RE: Final = re.compile(r"^codex[-_ /]", re.IGNORECASE) + + +def is_codex_user_agent(user_agent: str) -> bool: + return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index da033dc2276..ad4687e95db 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -31,6 +31,7 @@ from litellm.constants import ( SESSION_ID_OMITTED_METADATA_KEY, X_LITELLM_DISABLE_CALLBACKS, ) +from litellm.litellm_core_utils.core_helpers import is_codex_user_agent from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( TRUSTED_CALLBACK_VARS_FIELD, @@ -83,10 +84,6 @@ _EXPLICIT_SESSION_HEADERS: Final = frozenset({"x-litellm-trace-id", "x-litellm-s # ``session-id``/``thread-id``; builds before the codex-api split sent # ``session_id``/``conversation_id``. Ordered session before thread. _CODEX_SESSION_ID_HEADERS: Final = ("session-id", "session_id", "thread-id", "conversation_id") -# Matches every first-party Codex originator: codex-tui, codex_cli_rs, codex_exec, -# codex_vscode, "Codex ...". A separator is required so an unrelated "codexfoo" client -# does not read as Codex. -_CODEX_CLIENT_PREFIX_RE: Final = re.compile(r"^codex[-_ /]", re.IGNORECASE) # Session-id values must be non-empty strings of alphanumerics, hyphens, or underscores # (covers UUIDs and most common session-id formats). _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") @@ -810,16 +807,6 @@ def apply_missing_session_id_policy( ) -def is_codex_user_agent(user_agent: str) -> bool: - """Codex builds its user agent as ``/ ...`` and ships - several first-party originators: ``codex-tui``, ``codex_cli_rs``, - ``codex_exec`` (exec mode), ``codex_vscode`` (IDE extension) and ``Codex ...`` - (see ``is_first_party_originator`` in codex-rs). They agree only on the - ``codex`` stem, and the TUI sends a bare ``codex-tui`` with no version at all, - so match the stem plus a separator rather than any one spelling.""" - return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) - - def should_auto_drop_params_for_agentic_cli(user_agent: str, data: dict, proxy_config: ProxyConfig) -> bool: """drop_params defaults to on for agentic CLIs so their client-specific params (e.g. Claude Code's thinking, Codex's service_tier) don't fail diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 3d2b33a2ecd..be501db04f7 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -446,7 +446,9 @@ Reasoning markers in the system prompt do **not** trigger the reasoning override Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead -By default the router strips complete ``, ``, ``, ``, and `` blocks. It also strips Codex repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path +By default the router strips complete `` blocks. For requests with a Codex user agent, it also strips complete ``, ``, ``, and `` blocks, plus repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path. Other clients keep those tags and their contents + +The proxy records the incoming user agent in request metadata. SDK callers can supply `metadata.user_agent` (or `litellm_metadata.user_agent` on Responses requests), or configure `reminder_markers` explicitly when their client identity is unavailable The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 923aac85ee2..4c63452c1e9 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -39,6 +39,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_metadata_variable_name_from_kwargs, + is_codex_user_agent, ) from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -386,8 +387,8 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | N _REMINDER_OPEN: Final = "" _REMINDER_CLOSE: Final = "" -_DEFAULT_REMINDER_MARKERS: Final = ( - (_REMINDER_OPEN, _REMINDER_CLOSE), +_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),) +_CODEX_REMINDER_MARKERS: Final = _DEFAULT_REMINDER_MARKERS + ( ("", ""), ("", ""), ("", ""), @@ -1951,6 +1952,7 @@ class ComplexityRouter(CustomLogger): raise ValueError("classifier_llm_config is not set") include_assistant: Final = self.config.classifier_context_include_assistant_turns + marker_pairs: Final = self._reminder_markers_for_request(request_kwargs or {}) context_enabled: Final = bool(messages) and self.config.classifier_context_window_size > 0 prior_turns: Final = ( _extract_prior_turns( @@ -1960,20 +1962,14 @@ class ComplexityRouter(CustomLogger): budget_chars=self.config.classifier_context_budget_chars, per_turn_chars=self.config.classifier_context_per_turn_chars, include_assistant=include_assistant, - marker_pairs=self._reminder_markers, + marker_pairs=marker_pairs, ) if context_enabled else () ) has_prior_conversation: Final = ( context_enabled - and len( - tuple( - islice( - _iter_context_turns_newest_first(messages or (), include_assistant, self._reminder_markers), 2 - ) - ) - ) + and len(tuple(islice(_iter_context_turns_newest_first(messages or (), include_assistant, marker_pairs), 2))) > 1 ) @@ -2434,7 +2430,7 @@ class ComplexityRouter(CustomLogger): body if isinstance(body, Mapping) else None, resolved_messages, tuple(self.config.plan_mode_patterns or ()), - self._reminder_markers, + self._reminder_markers_for_request(request_kwargs), ) def _matched_housekeeping_sentinel(self, newest_ask: str | None) -> str | None: @@ -3255,6 +3251,18 @@ class ComplexityRouter(CustomLogger): """ return _extract_current_ask_and_system_prompt(messages) + def _reminder_markers_for_request(self, request_kwargs: Mapping[str, object]) -> tuple[tuple[str, str], ...]: + if self.config.reminder_markers is not None: + return self._reminder_markers + if any( + is_codex_user_agent(user_agent) + for metadata_key in ("litellm_metadata", "metadata") + if isinstance(metadata := request_kwargs.get(metadata_key), Mapping) + if isinstance(user_agent := metadata.get("user_agent"), str) + ): + return _CODEX_REMINDER_MARKERS + return _DEFAULT_REMINDER_MARKERS + @staticmethod def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]: """Metadata may land on `metadata` or `litellm_metadata` depending on the @@ -3356,6 +3364,7 @@ class ComplexityRouter(CustomLogger): # chat-completions messages, so it is real work on every non-chat surface, and # both the conversation shape and the classifier read the same list. resolved_messages: Final = self._resolve_messages(messages, request_kwargs) + marker_pairs: Final = self._reminder_markers_for_request(request_kwargs) conversation_continuing: Final = _conversation_is_continuing(resolved_messages) use_session_affinity: Final = self._uses_tier_pin @@ -3365,7 +3374,7 @@ class ComplexityRouter(CustomLogger): # In 'user_turn' mode a held pin is replayed only on continuation turns; a new human # ask falls through and re-classifies. session_affinity restores pin-first for asks too. pin_replay_allowed: Final = bool(self.config.session_affinity) or not _newest_turn_is_human_ask( - resolved_messages, self._reminder_markers + resolved_messages, marker_pairs ) if cache_key is not None and pin_replay_allowed: @@ -3376,7 +3385,7 @@ class ComplexityRouter(CustomLogger): pin_escalation_keyword: str | None = None if self.escalation_keywords: user_message: Final = ( - _newest_turn_ask(resolved_messages, self._reminder_markers) if resolved_messages else None + _newest_turn_ask(resolved_messages, marker_pairs) if resolved_messages else None ) if user_message is not None: pin_escalation_keyword = self._matched_escalation_keyword(user_message) @@ -3564,7 +3573,8 @@ class ComplexityRouter(CustomLogger): # Determine whether the original request used messages directly has_original_messages: Final = messages is not None and len(messages) > 0 - user_message, system_prompt = _extract_current_ask_and_system_prompt(resolved_messages, self._reminder_markers) + marker_pairs: Final = self._reminder_markers_for_request(request_kwargs) + user_message, system_prompt = _extract_current_ask_and_system_prompt(resolved_messages, marker_pairs) classifier_images: Final = self._classifier_image_parts(resolved_messages) if user_message is None and not classifier_images: @@ -3598,7 +3608,7 @@ class ComplexityRouter(CustomLogger): ) ask: Final = user_message or "" - newest_ask: Final = _newest_turn_ask(resolved_messages, self._reminder_markers) + newest_ask: Final = _newest_turn_ask(resolved_messages, marker_pairs) escalation_keyword: Final = self._matched_escalation_keyword(newest_ask) if newest_ask is not None else None # Resolved here rather than beside the classifier because the keyword-override path below # returns before any classification runs, and a forced tier gets stuck for the same reason diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 6ab4aa85148..fb7003c887c 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1292,8 +1292,9 @@ class ComplexityRouterConfig(BaseModel): "Override the delimiter pairs used to recognize and strip harness-injected reminder " "blocks before classification. A harness that wraps injected context differently per " "agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than " - "adds to, the built-in system-reminder and Codex envelope pairs, so list every " - "built-in pair your harness also emits. Matching is case-insensitive." + "adds to, the built-in system-reminder pair and the Codex envelope pairs enabled " + "for Codex user agents, so list every built-in pair your harness also emits. " + "Matching is case-insensitive." ), ) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 792963fde23..44d7454e836 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -7625,9 +7625,38 @@ _CODEX_ENVELOPES: Final = ( class TestContextAwareClassifier: """Test the new classifier context window and trajectory signals.""" + @pytest.mark.asyncio + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) + @pytest.mark.parametrize("user_agent", (None, "curl/8.7.1", "codexify/1.0")) + async def test_non_codex_requests_preserve_tagged_asks(self, envelope: str, user_agent: str | None) -> None: + completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}')) + router: Final = ComplexityRouter( + model_name="router", + litellm_router_instance=MagicMock(acompletion=completion), + complexity_router_config={ + "tiers": {"COMPLEX": "task-model"}, + "default_model": "fallback-model", + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "escalation_keywords": [], + }, + ) + + response: Final = await router.async_pre_routing_hook( + model="router", + request_kwargs={"metadata": {"user_agent": user_agent}} if user_agent is not None else {}, + messages=[{"role": "user", "content": envelope}], + ) + + assert response is not None + assert response.model == "task-model" + completion.assert_awaited_once() + assert completion.call_args.kwargs["messages"][1]["content"].strip() == f"Classify this message:\n{envelope}" + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) def test_codex_envelopes_preserve_delegated_task_and_prior_context(self, envelope: str) -> None: from litellm.router_strategy.complexity_router.complexity_router import ( + _CODEX_REMINDER_MARKERS, _extract_current_ask_and_system_prompt, _extract_prior_turns, _newest_turn_ask, @@ -7644,21 +7673,25 @@ class TestContextAwareClassifier: {"role": "user", "content": envelope}, ] - assert _extract_current_ask_and_system_prompt(messages)[0] == _CODEX_NEW_TASK - assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False) == ( + assert _extract_current_ask_and_system_prompt(messages, _CODEX_REMINDER_MARKERS)[0] == _CODEX_NEW_TASK + assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False, _CODEX_REMINDER_MARKERS) == ( ("user", "Design cache invalidation"), ) - assert _newest_turn_ask(messages) is None - assert _newest_turn_is_human_ask(messages) is False - assert _extract_current_ask_and_system_prompt([messages[-1]])[0] is None + assert _newest_turn_ask(messages, _CODEX_REMINDER_MARKERS) is None + assert _newest_turn_is_human_ask(messages, _CODEX_REMINDER_MARKERS) is False + assert _extract_current_ask_and_system_prompt([messages[-1]], _CODEX_REMINDER_MARKERS)[0] is None @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) def test_codex_marker_override_and_incomplete_blocks_preserve_text(self, envelope: str) -> None: - from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + from litellm.router_strategy.complexity_router.complexity_router import ( + _CODEX_REMINDER_MARKERS, + _strip_reminder_blocks, + ) incomplete: Final = envelope.rsplit("noise{envelope}", (("", ""),)) == envelope @pytest.mark.asyncio @@ -7681,9 +7714,12 @@ class TestContextAwareClassifier: ] original: Final = deepcopy(messages) request_kwargs: Final = ( - {"input": messages, "litellm_metadata": {"user_api_key_request_route": "/v1/responses"}} + { + "input": messages, + "litellm_metadata": {"user_api_key_request_route": "/v1/responses", "user_agent": "codex-tui"}, + } if responses_api - else {} + else {"metadata": {"user_agent": "codex-tui"}} ) response: Final = await router.async_pre_routing_hook( @@ -7706,6 +7742,46 @@ class TestContextAwareClassifier: else: assert response.messages == original + @pytest.mark.asyncio + @pytest.mark.parametrize("custom_markers", (False, True)) + async def test_codex_markers_are_request_scoped_and_respect_overrides(self, custom_markers: bool) -> None: + completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}')) + router: Final = ComplexityRouter( + model_name="router", + litellm_router_instance=MagicMock(acompletion=completion), + complexity_router_config={ + "tiers": {"COMPLEX": "task-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "classifier_context_window_size": 2, + "escalation_keywords": [], + **({"reminder_markers": [{"open": "", "close": ""}]} if custom_markers else {}), + }, + ) + envelope: Final = "\n".join(_CODEX_ENVELOPES) + prior: Final = f"{envelope}\nDesign cache invalidation" + messages: Final = [ + {"role": "user", "content": prior}, + {"role": "user", "content": _CODEX_NEW_TASK}, + {"role": "user", "content": envelope}, + ] + for user_agent in ("codex-tui", "curl/8.7.1", "codex_cli_rs/0.62.0"): + response: Final = await router.async_pre_routing_hook( + model="router", request_kwargs={"metadata": {"user_agent": user_agent}}, messages=messages + ) + assert response is not None + assert response.model == "task-model" + payload: Final = completion.call_args.kwargs["messages"][1]["content"] + if user_agent.startswith("codex") and not custom_markers: + assert payload.endswith(f"Classify this message:\n{_CODEX_NEW_TASK}") + assert "Design cache invalidation" in payload + assert "LITELLM ESCALATE" not in payload + else: + assert payload.endswith(f"Classify this message:\n{envelope}") + assert prior in payload + assert response.messages == messages + assert completion.await_count == 3 + @pytest.mark.parametrize( "messages,expected_ask", [ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5c4661ab357..ab2c1f27a59 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35221,7 +35221,7 @@ export interface components { reasoning_override_min_score?: number | null; /** * Reminder Markers - * @description Override the delimiter pairs used to recognize and strip harness-injected reminder blocks before classification. A harness that wraps injected context differently per agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than adds to, the built-in default of ('', ''), so a harness that also emits that pair lists it too. Matching is case-insensitive. + * @description Override the delimiter pairs used to recognize and strip harness-injected reminder blocks before classification. A harness that wraps injected context differently per agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than adds to, the built-in system-reminder pair and the Codex envelope pairs enabled for Codex user agents, so list every built-in pair your harness also emits. Matching is case-insensitive. */ reminder_markers?: components["schemas"]["ReminderMarkerPair"][] | null; /** From 41a147f2a0abfe1455e9560252c090d175d650d3 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:22:17 -0700 Subject: [PATCH 333/411] fix(router): normalize Azure classifier audit payloads --- litellm/llms/azure/azure.py | 2 ++ .../test_litellm_logging.py | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 46a9dd1a531..587165e6991 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -323,6 +323,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): "api_version": api_version, "api_base": api_base, "complete_input_dict": data, + "openai_sdk": True, }, ) if not isinstance(max_retries, int): @@ -429,6 +430,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): "api_base": api_base, "acompletion": True, "complete_input_dict": data, + "openai_sdk": True, }, ) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 28545bb2349..6de0fd3d4e0 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6804,11 +6804,11 @@ def test_get_error_information_redacts_provider_key_from_upstream_url(): @pytest.mark.asyncio -@pytest.mark.parametrize("provider", ["openai", "anthropic", "bedrock"]) +@pytest.mark.parametrize("provider", ["openai", "azure", "anthropic", "bedrock"]) async def test_classifier_audit_matches_provider_transport(provider: str) -> None: import json - from openai import AsyncOpenAI + from openai import AsyncAzureOpenAI, AsyncOpenAI from litellm.litellm_core_utils.classifier_logging import classifier_input_snapshot from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -6844,9 +6844,17 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non handler: Final = AsyncHTTPHandler() await handler.close() handler.client = http_client - client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client) if provider == "openai" else handler + client: Final = ( + AsyncAzureOpenAI( + api_key="transport-only", azure_endpoint="https://azure.invalid", + api_version="2025-04-01-preview", http_client=http_client, + ) + if provider == "azure" else AsyncOpenAI(api_key="transport-only", http_client=http_client) + if provider == "openai" else handler + ) model: Final = { "openai": "openai/gpt-5.6", + "azure": "azure/gpt-5.6", "anthropic": "anthropic/claude-haiku-4-5", "bedrock": "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", }[provider] @@ -6859,8 +6867,9 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, success_callback=[capture], num_retries=0, + **({"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} if provider == "azure" else {}), **({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}} - if provider == "openai" else {}), + if provider in ("openai", "azure") else {}), ) await asyncio.gather(run("request-one"), run("request-two")) @@ -6877,7 +6886,7 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non marker: Final = "request-one" if "request-one" in json.dumps(snapshot) else "request-two" assert payload["originating_request_masked"] == {"input": f"source-only-{marker}"} assert classifier_input_snapshot(snapshot) is not None - if provider != "openai": + if provider not in ("openai", "azure"): assert all("system" in request for request in requests) From 181b3fd94a17e54f0acbf42776bef36b5da4f2ed Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:24:42 -0700 Subject: [PATCH 334/411] fix(router): classify new asks before reminder-only tails --- .../complexity_router/README.md | 2 + .../complexity_router/complexity_router.py | 32 +++++-- .../router_strategy/test_complexity_router.py | 96 +++++++++++++++++++ 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index be501db04f7..f92e3569bbe 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -452,6 +452,8 @@ The proxy records the incoming user agent in request metadata. SDK callers can s The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content +In `classification_mode: user_turn`, complete text-only reminder tails leave the preceding fresh ask eligible for classification. Assistant turns and tool results still mark continuations, including tool results carried alongside reminder text + `reminder_markers` replaces these defaults with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: ```yaml diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 4c63452c1e9..a03b01ff1e0 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -599,6 +599,18 @@ def _last_human_ask_index( ) +def _is_reminder_only_turn(message: Mapping[str, object], marker_pairs: tuple[tuple[str, str], ...]) -> bool: + if message.get("role") != "user": + return False + content: Final = message.get("content") + if not isinstance(content, str) and not ( + isinstance(content, list) and all(isinstance(part, Mapping) and part.get("type") == "text" for part in content) + ): + return False + text: Final = _message_text(content) + return bool(text.strip()) and not _strip_reminder_blocks(text, marker_pairs) + + def _newest_turn_is_human_ask( messages: Sequence[Mapping[str, object]] | None, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, @@ -609,21 +621,23 @@ def _newest_turn_is_human_ask( Anchored on `_last_human_ask_index` so every surface's plumbing reads as a continuation: chat-completions tool turns are role=tool, Messages-surface tool_result turns flatten to empty human text, and a hybrid turn carrying an ask alongside a tool_result still counts as an ask. - Compared against the newest non-system message rather than the raw tail, because Claude Code - appends a system-role reminder after the human turn; that trailing plumbing is neither an ask - nor loop traffic and must not turn a fresh ask into a continuation. An unreadable request (no - messages) is treated as a continuation: there is no ask to classify, which is the same reading - `_extract_current_ask_and_system_prompt` gives it downstream. + Trailing system messages and complete text-only reminders do not turn a fresh ask into a + continuation. Assistant turns and non-text content, including tool results alongside reminders, + still form continuation boundaries. An unreadable request has no ask to classify. """ if not messages: return False - newest_non_system: Final = next( - (index for index in range(len(messages) - 1, -1, -1) if messages[index].get("role") != "system"), + newest_activity: Final = next( + ( + index + for index in range(len(messages) - 1, -1, -1) + if messages[index].get("role") != "system" and not _is_reminder_only_turn(messages[index], marker_pairs) + ), None, ) - if newest_non_system is None: + if newest_activity is None: return False - return _last_human_ask_index(messages, marker_pairs) == newest_non_system + return _last_human_ask_index(messages, marker_pairs) == newest_activity def _iter_system_scope_texts( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 44d7454e836..750b4780c04 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -7625,6 +7625,102 @@ _CODEX_ENVELOPES: Final = ( class TestContextAwareClassifier: """Test the new classifier context window and trajectory signals.""" + @pytest.mark.parametrize( + "tail,expected", + ( + ([{"role": "user", "content": [{"type": "text", "text": _CODEX_ENVELOPES[0]}]}], True), + ([{"role": "assistant", "content": _CODEX_ENVELOPES[0]}], False), + ([{"role": "tool", "content": _CODEX_ENVELOPES[0]}], False), + ([{"role": "user", "content": " "}], False), + ( + [{"role": "user", "content": [_TOOL_RESULT, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}], + False, + ), + ( + [{"role": "user", "content": [{"type": "image_url"}, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}], + False, + ), + ), + ) + def test_only_text_reminder_tails_are_ignored_for_new_asks(self, tail: list[dict[str, object]], expected: bool) -> None: + from litellm.router_strategy.complexity_router.complexity_router import ( + _CODEX_REMINDER_MARKERS, + _newest_turn_is_human_ask, + ) + + assert _newest_turn_is_human_ask([_ASKED, *tail], _CODEX_REMINDER_MARKERS) is expected + assert _newest_turn_is_human_ask(tail, _CODEX_REMINDER_MARKERS) is False + + @pytest.mark.asyncio + @pytest.mark.parametrize("new_ask", (_CODEX_NEW_TASK, "Now design cache invalidation")) + @pytest.mark.parametrize("responses_api", (False, True)) + @pytest.mark.parametrize("session_affinity", (False, True)) + async def test_codex_tail_preserves_new_ask_and_tool_continuation_boundaries( + self, new_ask: str, responses_api: bool, session_affinity: bool + ) -> None: + completion: Final = AsyncMock( + side_effect=[_llm_response('{"tier":"SIMPLE"}'), _llm_response('{"tier":"COMPLEX"}')] + ) + router: Final = ComplexityRouter( + model_name="router", + litellm_router_instance=MagicMock(acompletion=completion, cache=DualCache()), + complexity_router_config={ + "tiers": {"SIMPLE": "simple-model", "COMPLEX": "task-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "classification_mode": "user_turn", + "session_affinity": session_affinity, + "escalation_keywords": [], + }, + ) + metadata: Final = {"user_agent": "codex-tui", "session_id": "codex-tail-session"} + first_messages: Final = [{"role": "user", "content": "Hello"}] + tail: Final = [{"role": "user", "content": envelope} for envelope in _CODEX_ENVELOPES] + new_messages: Final = [ + *first_messages, + {"role": "assistant", "content": "Hello"}, + {"role": "user", "content": new_ask}, + *tail, + ] + continuation: Final = [ + *new_messages, + {"role": "assistant", "content": "Working on it"}, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "read-cache", "content": "cache source"}, + {"type": "text", "text": _CODEX_ENVELOPES[0]}, + ], + }, + *tail, + ] + results: Final = [ + await router.async_pre_routing_hook( + model="router", + request_kwargs=( + {"input": messages, "litellm_metadata": {**metadata, "user_api_key_request_route": "/v1/responses"}} + if responses_api + else {"metadata": metadata} + ), + messages=None if responses_api else messages, + input=messages if responses_api else None, + ) + for messages in (first_messages, new_messages, continuation) + ] + + assert [result.model for result in results] == ( + ["simple-model", "simple-model", "simple-model"] + if session_affinity + else ["simple-model", "task-model", "task-model"] + ) + assert completion.await_count == (1 if session_affinity else 2) + assert results[-1].routing_decision["cause"] == ( + "session_affinity_pin" if session_affinity else "user_turn_continuation" + ) + if not session_affinity: + assert completion.call_args.kwargs["messages"][1]["content"].endswith(f"Classify this message:\n{new_ask}") + assert results[1].messages == (None if responses_api else new_messages) + @pytest.mark.asyncio @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) @pytest.mark.parametrize("user_agent", (None, "curl/8.7.1", "codexify/1.0")) From f4ebcef0a17c2952ab90100fdba702d863c728f9 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:32:09 -0700 Subject: [PATCH 335/411] fix(router): classify encrypted delegated tasks with native Responses --- .../transformation.py | 3 + .../complexity_router/README.md | 10 + .../complexity_router/complexity_router.py | 119 ++++++++++-- .../router_strategy/test_complexity_router.py | 178 ++++++++++++++++++ 4 files changed, 293 insertions(+), 17 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index c3c18e3d009..5a6debc4af5 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1201,6 +1201,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Cast to Any to match the expected union type for tools list items tools.append(cast(Any, web_search_tool)) + def transform_response_format_to_text_format(self, response_format: object) -> "ResponseText | None": + return self._transform_response_format_to_text_format(response_format) + def _transform_response_format_to_text_format(self, response_format: object) -> "ResponseText | None": """ Transform Chat Completion response_format parameter to Responses API text.format parameter. diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index d605b43e42a..79119008aa2 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -361,6 +361,16 @@ model_list: keep the classifier deployment or provider default, or set a supported value such as `none` or `low` to override that call. +When the current ask is a Responses API `agent_message` containing `encrypted_content`, LLM +classification preserves the encrypted task and uses native Responses. This also bypasses the +local scoring shortcut in `heuristic_first` and `hybrid` modes. The configured classifier must use +a native OpenAI or Azure OpenAI Responses deployment with access to the encrypted content. The +provider handles the encrypted task, and the classifier still chooses the tier dynamically + +Unsupported classifier deployments and provider decryption errors use the existing +`classifier_fallback` policy. No fixed tier is introduced for encrypted tasks. Plaintext asks and +requests carrying only historical encrypted reasoning retain the existing classifier path + Classifier calls have a one-attempt hard deadline. After a timeout, the router opens a process-local circuit for that classifier and sends every session through `classifier_fallback` for `classifier_llm_config.circuit_breaker_cooldown_seconds` (30 seconds by default). When the cooldown diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index faafcea404a..3babe5b96ed 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -25,7 +25,7 @@ from threading import Lock from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast -from pydantic import BaseModel, create_model +from pydantic import BaseModel, TypeAdapter, create_model from litellm._logging import verbose_router_logger from litellm.constants import ( @@ -56,6 +56,7 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionImageObject, ChatCompletionTextObject, + ResponsesAPIResponse, ) from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, @@ -364,7 +365,7 @@ def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[ return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} -def _response_cost_or_none(response: ModelResponse) -> float | None: +def _response_cost_or_none(response: ModelResponse | ResponsesAPIResponse) -> float | None: hidden_params: Final = response._hidden_params if not isinstance(hidden_params, dict): return None @@ -486,6 +487,36 @@ def _human_text(content: object, marker_pairs: tuple[tuple[str, str], ...] = _DE return _strip_reminder_blocks(_message_text(content), marker_pairs) +def _encrypted_classifier_task( + request_kwargs: Mapping[str, object] | None, + marker_pairs: tuple[tuple[str, str], ...], +) -> dict[str, object] | None: + from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages + + raw_input: Final = (request_kwargs or EMPTY_MAPPING).get("input") + if not isinstance(raw_input, list) or (request_kwargs or EMPTY_MAPPING).get("messages"): + return None + items: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(raw_input) + current: Final = next( + ( + item + for item in reversed(items) + if (messages := resolve_structured_messages(messages=None, request_kwargs={"input": [item]})) + and any(_iter_human_asks_newest_first(messages, marker_pairs)) + ), + None, + ) + if current is None or current.get("type") != "agent_message" or not isinstance(current.get("content"), list): + return None + parts: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(current["content"]) + if not any(part.get("type") == "encrypted_content" and part.get("encrypted_content") for part in parts): + return None + return { + **current, + "content": [part for part in parts if part.get("type") in ("input_text", "encrypted_content")], + } + + def _iter_human_asks_newest_first( messages: Sequence[Mapping[str, object]], marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, @@ -1630,6 +1661,10 @@ class ComplexityRouter(CustomLogger): return self._classify_with_heuristic_v2(prompt) if self.config.classifier_type == "custom": return await self._classify_with_plugin(prompt, system_prompt, request_kwargs, raw_messages) + if self.config.classifier_type in ("heuristic_first", "hybrid") and _encrypted_classifier_task( + request_kwargs, self._reminder_markers + ): + return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages) if self.config.classifier_type == "heuristic_first" and self.config.classifier_llm_config is not None: return await self._classify_heuristic_first(prompt, system_prompt, request_kwargs, messages) if self.config.classifier_type == "hybrid" and self.config.classifier_llm_config is not None: @@ -1970,8 +2005,9 @@ class ComplexityRouter(CustomLogger): > 1 ) + encrypted_task: Final = _encrypted_classifier_task(request_kwargs, self._reminder_markers) user_payload: Final = self._build_classifier_user_payload( - prompt=prompt, + prompt="The delegated task in the following agent_message." if encrypted_task is not None else prompt, system_prompt=system_prompt, prior_turns=prior_turns, messages=messages, @@ -2004,34 +2040,37 @@ class ComplexityRouter(CustomLogger): if llm_config.reasoning_effort is not None: classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort}) - proxy_server_request: Final = { - "body": { - "model": llm_config.model, - "messages": messages_for_call, - "response_format": response_format, - **classifier_call_params, - } - } + payload: Final = ( + self._native_classifier_payload(llm_config.model, messages_for_call, response_format, encrypted_task) + if encrypted_task is not None + else {"messages": messages_for_call, "response_format": response_format, **classifier_call_params} + ) + proxy_server_request: Final = {"body": {"model": llm_config.model, **payload}} + classify: Final = ( + self.litellm_router_instance.aresponses + if encrypted_task is not None + else self.litellm_router_instance.acompletion + ) classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000 - response: Final[ModelResponse] = await asyncio.wait_for( - self.litellm_router_instance.acompletion( + response: Final[ModelResponse | ResponsesAPIResponse] = await asyncio.wait_for( + classify( model=llm_config.model, - messages=messages_for_call, stream=False, - response_format=response_format, timeout=classifier_timeout_s, num_retries=0, disable_fallbacks=True, metadata=metadata, proxy_server_request=proxy_server_request, turn_off_message_logging=turn_off_message_logging, - **classifier_call_params, + **payload, **_parent_session_kwargs(request_kwargs), ), timeout=classifier_timeout_s, ) - content: Final = response.choices[0].message.content + content: Final = ( + response.output_text if isinstance(response, ResponsesAPIResponse) else response.choices[0].message.content + ) if not content: raise ValueError("LLM classifier returned empty content") raw_tier: Final = _LabeledTierClassification.model_validate_json(content).tier @@ -2040,6 +2079,52 @@ class ComplexityRouter(CustomLogger): raise ValueError(f"LLM classifier returned an unrecognized tier: {raw_tier!r}") return tier, _response_cost_or_none(response) + def _native_classifier_payload( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: existing transformation accepts the SDK message list + response_format: Mapping[str, object], + encrypted_task: Mapping[str, object], + ) -> Mapping[str, object]: + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider + from litellm.types.router import LiteLLM_Params + + deployments: Final = self._group_deployments(model) + if not deployments: + raise ValueError("Encrypted task classification requires a native OpenAI Responses classifier deployment") + for params in (LiteLLM_Params.model_validate(deployment.get("litellm_params")) for deployment in deployments): + if declared_authenticating_provider(params.model, params.custom_llm_provider): + raise ValueError( + "Encrypted task classification requires a native OpenAI Responses classifier deployment" + ) + _, provider, _, _ = get_llm_provider(model=params.model, litellm_params=params) + if ( + provider not in ("openai", "azure") + or params.use_chat_completions_api + or params.model.startswith("openai/chat_completions/") + ): + raise ValueError( + "Encrypted task classification requires a native OpenAI Responses classifier deployment" + ) + transformation: Final = LiteLLMResponsesTransformationHandler() + input_items, instructions = transformation.convert_chat_completion_messages_to_responses_api(messages) + llm_config: Final = self.config.classifier_llm_config + reasoning: Final = ( + {"reasoning": {"effort": llm_config.reasoning_effort}} + if llm_config is not None and llm_config.reasoning_effort is not None + else {} + ) + return { + "input": [*input_items, encrypted_task], + "instructions": instructions, + "text": transformation.transform_response_format_to_text_format(dict(response_format)), + "store": False, + **reasoning, + } + @staticmethod def _build_classifier_user_payload( prompt: str, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 51103297c58..87438ef5c69 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5,6 +5,8 @@ Tests the rule-based complexity scoring and tier assignment logic. """ import asyncio +import copy +import json import logging import sys import time @@ -66,6 +68,7 @@ from litellm.types.router import ( LiteLLM_Params, TaggedPreRoutingStrategy, ) +from litellm.types.llms.openai import ResponsesAPIResponse requires_semantic_router = pytest.mark.skipif( @@ -2482,6 +2485,181 @@ class TestTierLabels: assert set(config.tier_boundaries) == {"simple_medium", "medium_complex", "complex_reasoning"} +def _encrypted_agent_task() -> dict[str, object]: + return { + "type": "agent_message", + "author": "/root", + "recipient": "/root/child", + "content": [ + {"type": "input_text", "text": "Message Type: NEW_TASK\nTask name: /root/child\nPayload:\nHello"}, + {"type": "encrypted_content", "encrypted_content": "opaque-provider-task"}, + ], + } + + +def _native_classifier_response(content: str) -> ResponsesAPIResponse: + response: Final = ResponsesAPIResponse( + id="resp_classifier", + created_at=0, + status="completed", + output=[{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": content}]}], + ) + response._hidden_params = {"response_cost": 0.0001} + return response + + +def _native_classifier_router( + output: str = '{"tier":"REASONING"}', + classifier_type: str = "llm", + deployment_model: str = "openai/gpt-6-astra", + failure: Exception | None = None, +) -> tuple[ComplexityRouter, MagicMock]: + dependency: Final = MagicMock( + aresponses=AsyncMock(return_value=_native_classifier_response(output), side_effect=failure), + acompletion=AsyncMock(return_value=_llm_response('{"tier":"SIMPLE"}')), + get_model_list=MagicMock(return_value=[{"litellm_params": {"model": deployment_model}}]), + ) + return ( + ComplexityRouter( + model_name="encrypted-router", + litellm_router_instance=dependency, + complexity_router_config={ + "tiers": {"SIMPLE": "cheap-model", "REASONING": "deep-model"}, + "classifier_type": classifier_type, + "classifier_llm_config": {"model": "classifier", "timeout_ms": 100, "reasoning_effort": "low"}, + "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None, + "hybrid_boundary_margin": 0.01 if classifier_type == "hybrid" else None, + "classifier_fallback": "default_model", + "default_model": "deep-model", + "session_affinity": False, + "deployment_affinity": False, + }, + ), + dependency, + ) + + +class TestEncryptedTaskClassifier: + @pytest.mark.asyncio + @pytest.mark.parametrize("classifier_type", ["llm", "heuristic_first", "hybrid"]) + @pytest.mark.parametrize("tier,model", [("SIMPLE", "cheap-model"), ("REASONING", "deep-model")]) + async def test_encrypted_task_routes_by_native_verdict(self, classifier_type: str, tier: str, model: str): + router, dependency = _native_classifier_router(json.dumps({"tier": tier}), classifier_type) + task: Final = _encrypted_agent_task() + request: Final = { + "input": [ + {"role": "user", "content": "Prior task context"}, + task, + {"type": "function_call_output", "call_id": "call_1", "output": "Tool output"}, + {"role": "user", "content": "Injected reminder"}, + ], + "instructions": "Caller constraints", + "tools": [{"type": "function", "name": "execute"}], + "previous_response_id": "resp_parent", + "litellm_session_id": "parent-session", + "litellm_trace_id": "parent-trace", + "turn_off_message_logging": True, + "litellm_metadata": {"user_api_key_hash": "caller-key-hash"}, + } + original: Final = copy.deepcopy(request) + + result: Final = await router.async_pre_routing_hook(model="encrypted-router", request_kwargs=request) + + assert result.model == model + assert result.routing_decision["tier"] == tier + assert result.routing_decision["cause"] == "llm_classifier" + assert result.routing_decision["classifier_cost"] == 0.0001 + assert result.messages is None + assert request == original + dependency.acompletion.assert_not_called() + call: Final = dependency.aresponses.call_args.kwargs + assert call["input"][-1] == task + assert "opaque-provider-task" not in json.dumps(call["input"][:-1]) + assert "Prior task context" in json.dumps(call["input"][:-1]) + assert "Caller constraints" in json.dumps(call["input"][:-1]) + assert "Caller constraints" not in call["instructions"] + assert "SIMPLE" in call["instructions"] and "REASONING" in call["instructions"] + assert call["text"]["format"]["schema"]["properties"]["tier"]["enum"] == [ + "SIMPLE", "MEDIUM", "COMPLEX", "REASONING" + ] + assert call["text"]["format"]["strict"] is True + assert call["reasoning"] == {"effort": "low"} + assert call["store"] is False + assert call["stream"] is False + assert "tools" not in call and "previous_response_id" not in call + assert "messages" not in call and "response_format" not in call + assert call["timeout"] == 0.1 and call["num_retries"] == 0 and call["disable_fallbacks"] is True + assert call["litellm_session_id"] == "parent-session" + assert call["litellm_trace_id"] == "parent-trace" + assert call["turn_off_message_logging"] is True + assert call["metadata"]["user_api_key_hash"] == "caller-key-hash" + assert call["proxy_server_request"]["body"]["input"] == call["input"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "items", + [ + [{"type": "reasoning", "encrypted_content": "opaque-history", "summary": []}, {"role": "user", "content": "hi"}], + [_encrypted_agent_task(), {"role": "user", "content": "hi"}], + [{**_encrypted_agent_task(), "content": [{"type": "input_text", "text": "hi"}]}], + [{"role": "user", "content": "gAAAA is plain text"}], + [{"role": "user", "content": "hi"}, {"type": "function_call_output", "call_id": "call_1", "output": "opaque-provider-task"}], + ], + ids=["historical-reasoning", "older-encrypted-task", "plaintext-agent", "ciphertext-looking-text", "tool-output"], + ) + async def test_other_asks_keep_chat_classifier(self, items: list[dict[str, object]]): + router, dependency = _native_classifier_router() + + result: Final = await router.async_pre_routing_hook(model="encrypted-router", request_kwargs={"input": items}) + + assert result.model == "cheap-model" + assert result.routing_decision["cause"] == "llm_classifier" + dependency.aresponses.assert_not_called() + dependency.acompletion.assert_awaited_once() + + @pytest.mark.asyncio + @pytest.mark.parametrize("output", ["", "not-json", '{"tier":"UNKNOWN"}']) + async def test_invalid_native_verdict_uses_existing_fallback(self, output: str): + router, dependency = _native_classifier_router(output=output) + + result: Final = await router.async_pre_routing_hook( + model="encrypted-router", request_kwargs={"input": [_encrypted_agent_task()]} + ) + + assert result.model == "deep-model" + assert result.routing_decision["cause"] == "default_model_fallback" + dependency.aresponses.assert_awaited_once() + dependency.acompletion.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("deployment_model", ["anthropic/test-classifier", "openai/chat_completions/gpt-6-astra"]) + async def test_incompatible_classifier_does_not_flatten_encryption(self, deployment_model: str): + router, dependency = _native_classifier_router(deployment_model=deployment_model) + + result: Final = await router.async_pre_routing_hook( + model="encrypted-router", request_kwargs={"input": [_encrypted_agent_task()]} + ) + + assert result.model == "deep-model" + assert result.routing_decision["cause"] == "default_model_fallback" + dependency.aresponses.assert_not_called() + dependency.acompletion.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("failure", [ValueError("invalid_encrypted_content"), TimeoutError("classifier timed out")]) + async def test_native_provider_failure_uses_existing_fallback(self, failure: Exception): + router, dependency = _native_classifier_router(failure=failure) + + result: Final = await router.async_pre_routing_hook( + model="encrypted-router", request_kwargs={"input": [_encrypted_agent_task()]} + ) + + assert result.model == "deep-model" + assert result.routing_decision["cause"] == "default_model_fallback" + dependency.aresponses.assert_awaited_once() + dependency.acompletion.assert_not_called() + + class TestLLMClassifier: """Test the LLM-based classifier path (aclassify) and its fallback behavior.""" From 33954d9cfe3eb887c26ab6a26dcd928ef213638a Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:35:59 -0700 Subject: [PATCH 336/411] refactor(router): validate classifier snapshots without recursion --- litellm/litellm_core_utils/classifier_logging.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/classifier_logging.py b/litellm/litellm_core_utils/classifier_logging.py index b9cb52a190d..c0c93161099 100644 --- a/litellm/litellm_core_utils/classifier_logging.py +++ b/litellm/litellm_core_utils/classifier_logging.py @@ -13,13 +13,13 @@ _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) def classifier_input_snapshot(value: object, *, openai_sdk: bool = False) -> Mapping[str, JsonValue] | None: - if openai_sdk and isinstance(value, Mapping): - body: Final = { - key: item for key, item in value.items() if key not in ("extra_headers", "extra_query", "extra_body") - } - extra_body: Final = value.get("extra_body") - return classifier_input_snapshot({**body, **extra_body} if isinstance(extra_body, Mapping) else body) try: + if openai_sdk and isinstance(value, Mapping): + body: Final = { + key: item for key, item in value.items() if key not in ("extra_headers", "extra_query", "extra_body") + } + extra_body: Final = value.get("extra_body") + return _JSON_OBJECT.validate_python({**body, **extra_body} if isinstance(extra_body, Mapping) else body) return ( _JSON_OBJECT.validate_json(value) if isinstance(value, (str, bytes)) From 83b1e9e0d7ac48e4806a91d1e1d7587725a82039 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:46:50 -0700 Subject: [PATCH 337/411] test(router): keep classifier E2E verification local --- tests/e2e/coverage_registry/reliability.yaml | 1 - tests/e2e/router/test_classifier_audit_e2e.py | 143 ------------------ 2 files changed, 144 deletions(-) delete mode 100644 tests/e2e/router/test_classifier_audit_e2e.py diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 4d40f19e67b..6b69677d490 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -1,5 +1,4 @@ # Reliability & Performance (behavior features). Grounded in litellm/router.py + router_strategy/ + router_utils/. -- {id: reliability.routing.classifier_audit.separates_provider_input_and_source, module: reliability, tier: P1, behavior: routing, variant: classifier_audit, assertions: [separates_provider_input_and_source], exercised_on: [chat_completions, messages, responses], source: "litellm/litellm_core_utils/classifier_logging.py", rationale: "Classifier spend details distinguish provider input from the credential-masked originating request across all three request surfaces"} - {id: reliability.fallback.5xx.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "5xx", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2024", rationale: "Reroute on provider 5xx to alternate deployment"} - {id: reliability.fallback.context_window.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: context_window, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6108", rationale: "Fallback when model exceeds context limit"} - {id: reliability.fallback.content_policy.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: content_policy, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6023", rationale: "Reroute on content-policy violation"} diff --git a/tests/e2e/router/test_classifier_audit_e2e.py b/tests/e2e/router/test_classifier_audit_e2e.py deleted file mode 100644 index 6b444441477..00000000000 --- a/tests/e2e/router/test_classifier_audit_e2e.py +++ /dev/null @@ -1,143 +0,0 @@ -import json -import os -from collections.abc import Iterator -from contextlib import ExitStack -from dataclasses import dataclass -from typing import Final, Literal - -import pytest -from pydantic import BaseModel, Field, JsonValue, TypeAdapter - -from e2e_config import unique_marker -from e2e_http import AnthropicHeaders, NoBody, unwrap -from models import ChatMessage, KeyGenerateBody, LiteLLMParamsBody -from proxy_client import ProxyClient - -pytestmark = pytest.mark.e2e - - -class AuditMetadata(BaseModel): - source_marker: str - authorization: str = "synthetic-audit-secret" - - -class AuditHeaders(AnthropicHeaders): - enable_redaction: str | None = Field(default=None, serialization_alias="x-litellm-enable-message-redaction") - - -class AuditRequest(BaseModel): - model: str - messages: list[ChatMessage] | None = None - system: str | None = None - instructions: str | None = None - input: str | None = None - max_tokens: int | None = None - max_output_tokens: int | None = None - metadata: AuditMetadata | None = None - litellm_metadata: AuditMetadata | None = None - - -class AuditResponse(BaseModel): - id: str - - -class AuditDetail(BaseModel): - proxy_server_request: dict[str, JsonValue] | str | None = None - response: dict[str, JsonValue] | str | None = None - - -@dataclass(frozen=True, slots=True) -class AuditDeployment: - alias: str - key: str - - -@pytest.fixture -def audit_deployment(proxy: ProxyClient, provider: str) -> Iterator[AuditDeployment]: - marker: Final = unique_marker() - classifier: Final = f"audit-classifier-{marker}" - target: Final = f"audit-target-{marker}" - alias: Final = f"audit-router-{marker}" - model: Final = os.environ.get( - f"E2E_CHEAP_{provider.upper()}_MODEL", "gpt-5.6" if provider == "openai" else "claude-haiku-4-5" - ) - params: Final = LiteLLMParamsBody( - model=f"{provider}/{model}", - api_key=os.environ.get(f"{provider.upper()}_API_KEY") or f"os.environ/{provider.upper()}_API_KEY", - api_base=os.environ.get(f"{provider.upper()}_API_BASE"), - ) - with ExitStack() as stack: - for name in (classifier, target): - stack.callback(proxy.delete_model, proxy.create_model(name, params)) - stack.callback(proxy.delete_model, proxy.create_model(alias, LiteLLMParamsBody( - model="auto_router/complexity_router", - complexity_router_config={ - "classifier_type": "llm", - "classifier_llm_config": {"model": classifier, "timeout_ms": 30000}, - "tiers": {tier: target for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")}, - }, - ))) - key: Final = proxy.generate_key(KeyGenerateBody(models=[alias, classifier, target])) - stack.callback(proxy.delete_key, key) - yield AuditDeployment(alias, key) - - -class TestClassifierAudit: - @pytest.mark.covers( - "reliability.routing.classifier_audit.separates_provider_input_and_source", - exercised_on=("chat_completions", "messages", "responses"), - ) - @pytest.mark.parametrize("surface", ["chat_completions", "messages", "responses"]) - @pytest.mark.parametrize("provider", ["openai", "anthropic"]) - @pytest.mark.parametrize("redact", [False, True]) - def test_classifier_audit_separates_input_and_masked_source( - self, proxy: ProxyClient, audit_deployment: AuditDeployment, surface: Literal["chat_completions", "messages", "responses"], - redact: bool, - ) -> None: - marker: Final = unique_marker() - source_marker: Final = f"source-only-{marker}" - prompt: Final = f"Reply with hello. Request label {marker}" - metadata: Final = AuditMetadata(source_marker=source_marker) - body: Final = AuditRequest( - model=audit_deployment.alias, - messages=[ChatMessage(role="user", content=prompt)] if surface != "responses" else None, - input=prompt if surface == "responses" else None, - system="Be concise" if surface == "messages" else None, - instructions="Be concise" if surface == "responses" else None, - max_tokens=128 if surface != "responses" else None, - max_output_tokens=128 if surface == "responses" else None, - metadata=metadata if surface != "responses" else None, - litellm_metadata=metadata if surface == "responses" else None, - ) - path: Final = {"chat_completions": "/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}[surface] - response: Final = unwrap(proxy.transport.post( - path, headers=AuditHeaders(authorization=f"Bearer {audit_deployment.key}", enable_redaction="true" if redact else None), - json=body, response_type=AuditResponse, - )) - assert response.id - rows: Final = proxy.poll_logs_for_key(audit_deployment.key, min_rows=2) - assert len(rows) == 2, "Expected a classifier spend row and a routed response spend row" - details: Final = tuple( - unwrap(proxy.transport.get( - f"/spend/logs/ui/{row.request_id}", headers=proxy.transport.master, - params=NoBody(), response_type=AuditDetail, - )) for row in rows - ) - adapter: Final = TypeAdapter(dict[str, JsonValue]) - requests: Final = tuple( - adapter.validate_json(detail.proxy_server_request) if isinstance(detail.proxy_server_request, str) - else detail.proxy_server_request or {} for detail in details - ) - audits: Final = tuple(item for item in requests if "classifier_input" in item) - if redact: - assert audits == () - assert all("originating_request_masked" not in item for item in requests) - return - assert len(audits) == 1, "The audit belongs only to the classifier call" - audit: Final = audits[0] - assert marker in json.dumps(audit["classifier_input"]) - assert source_marker not in json.dumps(audit["classifier_input"]) - assert source_marker in json.dumps(audit["originating_request_masked"]) - assert "synthetic-audit-secret" not in json.dumps(audit) - assert audit_deployment.alias in json.dumps(audit["originating_request_masked"]) - assert any("tier" in str(detail.response) for detail in details) From 208d554c008da91b74062413ad8d5d454f7746b5 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 13:04:36 -0700 Subject: [PATCH 338/411] fix(router): validate encrypted classifiers after deployment selection --- .../llms/base_llm/responses/transformation.py | 3 + .../llms/openai/responses/transformation.py | 3 + litellm/responses/main.py | 8 + .../complexity_router/README.md | 3 + .../complexity_router/complexity_router.py | 35 ++-- .../router_strategy/test_complexity_router.py | 151 ++++++++++++++++-- 6 files changed, 168 insertions(+), 35 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 1365941fe2a..14f00aaaa21 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -62,6 +62,9 @@ class BaseResponsesAPIConfig(ABC): """ return False + def supports_encrypted_agent_messages(self) -> bool: + return False + def sign_request( self, headers: dict, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 666e9b32011..833ae206024 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -110,6 +110,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def supports_native_file_search(self) -> bool: return True + def supports_encrypted_agent_messages(self) -> bool: + return self.custom_llm_provider in (LlmProviders.OPENAI, LlmProviders.AZURE) + @staticmethod def _is_gpt_5_model(model: str) -> bool: """Return True only for actual OpenAI GPT-5 models. diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 7a210cdd970..e88cd618a8b 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1078,6 +1078,7 @@ def responses( litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) _is_async: Final = kwargs.pop("aresponses", False) is True skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) + require_encrypted_task_support: Final = kwargs.pop("_require_encrypted_task_support", False) is True use_chat_completions_api = _pop_use_chat_completions_api_kw(kwargs) client_headers: Final = kwargs.get("headers") @@ -1186,6 +1187,13 @@ def responses( model, custom_llm_provider, deployment_model_info ) + if require_encrypted_task_support and ( + _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api) + or responses_api_provider_config is None + or not responses_api_provider_config.supports_encrypted_agent_messages() + ): + raise ValueError("Encrypted task classification requires a compatible native Responses deployment") + local_vars.update(kwargs) # Map reasoning_effort (from litellm_params/proxy config) to reasoning when not set if reasoning is None and "reasoning_effort" in local_vars: diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 267def7cc5b..38dfd143cbc 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -367,6 +367,9 @@ local scoring shortcut in `heuristic_first` and `hybrid` modes. The configured c a native OpenAI or Azure OpenAI Responses deployment with access to the encrypted content. The provider handles the encrypted task, and the classifier still chooses the tier dynamically +Compatibility is checked after normal deployment selection. A paused incompatible member of the +classifier group does not prevent an eligible compatible deployment from classifying the task + Unsupported classifier deployments and provider decryption errors use the existing `classifier_fallback` policy. No fixed tier is introduced for encrypted tasks. Plaintext asks and requests carrying only historical encrypted reasoning retain the existing classifier path diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 77de4df3cd7..6df74b26f60 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -25,7 +25,7 @@ from threading import Lock from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast -from pydantic import BaseModel, TypeAdapter, create_model +from pydantic import BaseModel, TypeAdapter, ValidationError, create_model from litellm._logging import verbose_router_logger from litellm.constants import ( @@ -504,7 +504,10 @@ def _encrypted_classifier_task( raw_input: Final = (request_kwargs or EMPTY_MAPPING).get("input") if not isinstance(raw_input, list) or (request_kwargs or EMPTY_MAPPING).get("messages"): return None - items: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(raw_input) + try: + items: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(raw_input) + except ValidationError: + return None current: Final = next( ( item @@ -516,7 +519,10 @@ def _encrypted_classifier_task( ) if current is None or current.get("type") != "agent_message" or not isinstance(current.get("content"), list): return None - parts: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(current["content"]) + try: + parts: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(current["content"]) + except ValidationError: + return None if not any(part.get("type") == "encrypted_content" and part.get("encrypted_content") for part in parts): return None return { @@ -2058,7 +2064,7 @@ class ComplexityRouter(CustomLogger): classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort}) payload: Final = ( - self._native_classifier_payload(llm_config.model, messages_for_call, response_format, encrypted_task) + self._native_classifier_payload(messages_for_call, response_format, encrypted_task) if encrypted_task is not None else {"messages": messages_for_call, "response_format": response_format, **classifier_call_params} ) @@ -2098,7 +2104,6 @@ class ComplexityRouter(CustomLogger): def _native_classifier_payload( self, - model: str, messages: list[AllMessageValues], # mutable-ok: existing transformation accepts the SDK message list response_format: Mapping[str, object], encrypted_task: Mapping[str, object], @@ -2106,26 +2111,7 @@ class ComplexityRouter(CustomLogger): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, ) - from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider - from litellm.types.router import LiteLLM_Params - deployments: Final = self._group_deployments(model) - if not deployments: - raise ValueError("Encrypted task classification requires a native OpenAI Responses classifier deployment") - for params in (LiteLLM_Params.model_validate(deployment.get("litellm_params")) for deployment in deployments): - if declared_authenticating_provider(params.model, params.custom_llm_provider): - raise ValueError( - "Encrypted task classification requires a native OpenAI Responses classifier deployment" - ) - _, provider, _, _ = get_llm_provider(model=params.model, litellm_params=params) - if ( - provider not in ("openai", "azure") - or params.use_chat_completions_api - or params.model.startswith("openai/chat_completions/") - ): - raise ValueError( - "Encrypted task classification requires a native OpenAI Responses classifier deployment" - ) transformation: Final = LiteLLMResponsesTransformationHandler() input_items, instructions = transformation.convert_chat_completion_messages_to_responses_api(messages) llm_config: Final = self.config.classifier_llm_config @@ -2139,6 +2125,7 @@ class ComplexityRouter(CustomLogger): "instructions": instructions, "text": transformation.transform_response_format_to_text_format(dict(response_format)), "store": False, + "_require_encrypted_task_support": True, **reasoning, } diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 780465dad3b..2ea8a8b53fe 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5,8 +5,10 @@ Tests the rule-based complexity scoring and tier assignment logic. """ import asyncio +from collections.abc import AsyncIterator import json from copy import deepcopy +from functools import partial import logging import sys import time @@ -14,6 +16,7 @@ from typing import Dict, Final, List from unittest.mock import AsyncMock, MagicMock, patch import pytest +import httpx from pydantic import ValidationError import litellm @@ -69,6 +72,7 @@ from litellm.types.router import ( TaggedPreRoutingStrategy, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler requires_semantic_router = pytest.mark.skipif( @@ -2520,11 +2524,21 @@ def _native_classifier_router( classifier_type: str = "llm", deployment_model: str = "openai/gpt-6-astra", failure: Exception | None = None, + native_router: Router | None = None, + http_handler: AsyncHTTPHandler | None = None, ) -> tuple[ComplexityRouter, MagicMock]: dependency: Final = MagicMock( - aresponses=AsyncMock(return_value=_native_classifier_response(output), side_effect=failure), + aresponses=( + native_router.factory_function(partial(litellm.aresponses, client=http_handler), call_type="aresponses") + if native_router is not None + else AsyncMock(return_value=_native_classifier_response(output), side_effect=failure) + ), acompletion=AsyncMock(return_value=_llm_response('{"tier":"SIMPLE"}')), - get_model_list=MagicMock(return_value=[{"litellm_params": {"model": deployment_model}}]), + get_model_list=( + native_router.get_model_list + if native_router is not None + else MagicMock(return_value=[{"litellm_params": {"model": deployment_model}}]) + ), ) return ( ComplexityRouter( @@ -2533,7 +2547,11 @@ def _native_classifier_router( complexity_router_config={ "tiers": {"SIMPLE": "cheap-model", "REASONING": "deep-model"}, "classifier_type": classifier_type, - "classifier_llm_config": {"model": "classifier", "timeout_ms": 100, "reasoning_effort": "low"}, + "classifier_llm_config": { + "model": "classifier", + "timeout_ms": 5000 if native_router is not None else 100, + "reasoning_effort": "low", + }, "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None, "hybrid_boundary_margin": 0.01 if classifier_type == "hybrid" else None, "classifier_fallback": "default_model", @@ -2546,6 +2564,18 @@ def _native_classifier_router( ) +@pytest.fixture +async def native_classifier_http() -> AsyncIterator[tuple[AsyncHTTPHandler, MagicMock]]: + respond: Final = MagicMock( + return_value=httpx.Response(200, json=_native_classifier_response('{"tier":"REASONING"}').model_dump()) + ) + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + handler.client = client + yield handler, respond + + class TestEncryptedTaskClassifier: @pytest.mark.asyncio @pytest.mark.parametrize("classifier_type", ["llm", "heuristic_first", "hybrid"]) @@ -2587,11 +2617,15 @@ class TestEncryptedTaskClassifier: assert "Caller constraints" not in call["instructions"] assert "SIMPLE" in call["instructions"] and "REASONING" in call["instructions"] assert call["text"]["format"]["schema"]["properties"]["tier"]["enum"] == [ - "SIMPLE", "MEDIUM", "COMPLEX", "REASONING" + "SIMPLE", + "MEDIUM", + "COMPLEX", + "REASONING", ] assert call["text"]["format"]["strict"] is True assert call["reasoning"] == {"effort": "low"} assert call["store"] is False + assert call["_require_encrypted_task_support"] is True assert call["stream"] is False assert "tools" not in call and "previous_response_id" not in call assert "messages" not in call and "response_format" not in call @@ -2606,13 +2640,25 @@ class TestEncryptedTaskClassifier: @pytest.mark.parametrize( "items", [ - [{"type": "reasoning", "encrypted_content": "opaque-history", "summary": []}, {"role": "user", "content": "hi"}], + [ + {"type": "reasoning", "encrypted_content": "opaque-history", "summary": []}, + {"role": "user", "content": "hi"}, + ], [_encrypted_agent_task(), {"role": "user", "content": "hi"}], [{**_encrypted_agent_task(), "content": [{"type": "input_text", "text": "hi"}]}], [{"role": "user", "content": "gAAAA is plain text"}], - [{"role": "user", "content": "hi"}, {"type": "function_call_output", "call_id": "call_1", "output": "opaque-provider-task"}], + [ + {"role": "user", "content": "hi"}, + {"type": "function_call_output", "call_id": "call_1", "output": "opaque-provider-task"}, + ], + ], + ids=[ + "historical-reasoning", + "older-encrypted-task", + "plaintext-agent", + "ciphertext-looking-text", + "tool-output", ], - ids=["historical-reasoning", "older-encrypted-task", "plaintext-agent", "ciphertext-looking-text", "tool-output"], ) async def test_other_asks_keep_chat_classifier(self, items: list[dict[str, object]]): router, dependency = _native_classifier_router() @@ -2639,9 +2685,28 @@ class TestEncryptedTaskClassifier: dependency.acompletion.assert_not_called() @pytest.mark.asyncio - @pytest.mark.parametrize("deployment_model", ["anthropic/test-classifier", "openai/chat_completions/gpt-6-astra"]) - async def test_incompatible_classifier_does_not_flatten_encryption(self, deployment_model: str): - router, dependency = _native_classifier_router(deployment_model=deployment_model) + @pytest.mark.parametrize( + "deployment_model", + ["anthropic/test-classifier", "openai/chat_completions/gpt-6-astra", "xai/test-classifier"], + ) + async def test_incompatible_classifier_does_not_flatten_encryption( + self, deployment_model: str, native_classifier_http: tuple[AsyncHTTPHandler, MagicMock] + ): + handler, respond = native_classifier_http + native: Final = Router( + model_list=[ + { + "model_name": "classifier", + "litellm_params": { + "model": deployment_model, + "api_key": "test-key", + "api_base": "https://classifier.test/v1", + }, + } + ], + num_retries=0, + ) + router, _ = _native_classifier_router(native_router=native, http_handler=handler) result: Final = await router.async_pre_routing_hook( model="encrypted-router", request_kwargs={"input": [_encrypted_agent_task()]} @@ -2649,8 +2714,72 @@ class TestEncryptedTaskClassifier: assert result.model == "deep-model" assert result.routing_decision["cause"] == "default_model_fallback" + respond.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("blocked", [True, False]) + async def test_native_classifier_validates_selected_deployment( + self, blocked: bool, native_classifier_http: tuple[AsyncHTTPHandler, MagicMock] + ): + handler, respond = native_classifier_http + native: Final = Router( + model_list=[ + { + "model_name": "classifier", + "litellm_params": {"model": "anthropic/test-classifier", "api_key": "test-key", "order": 0}, + "model_info": {"id": "incompatible", "blocked": blocked}, + }, + { + "model_name": "classifier", + "litellm_params": { + "model": "openai/gpt-6-astra", + "api_key": "test-key", + "order": 1, + "api_base": "https://classifier.test/v1", + }, + "model_info": {"id": "compatible"}, + }, + ], + num_retries=0, + ) + router, _ = _native_classifier_router(native_router=native, http_handler=handler) + task: Final = _encrypted_agent_task() + + result: Final = await router.async_pre_routing_hook(model="encrypted-router", request_kwargs={"input": [task]}) + + assert result.model == "deep-model" + assert result.routing_decision["cause"] == ("llm_classifier" if blocked else "default_model_fallback") + if blocked: + respond.assert_called_once() + request: Final = respond.call_args.args[0] + assert request.url.path == "/v1/responses" + body: Final = json.loads(request.content) + assert body["input"][-1] == task + assert "_require_encrypted_task_support" not in body + else: + respond.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("classifier_type", ["llm", "heuristic_first", "hybrid"]) + @pytest.mark.parametrize( + "input_items", + [ + ["unsupported-input-item"], + [{**_encrypted_agent_task(), "content": [{"type": "input_text", "text": "hi"}, None]}], + ], + ) + async def test_encrypted_detection_does_not_reject_other_input_shapes( + self, classifier_type: str, input_items: list[object] + ): + router, dependency = _native_classifier_router(classifier_type=classifier_type) + + result: Final = await router.aclassify("hi", request_kwargs={"input": input_items}) + + assert result.cause != "default_model_fallback" + assert result.tier == ComplexityTier.SIMPLE dependency.aresponses.assert_not_called() - dependency.acompletion.assert_not_called() + if classifier_type == "llm": + dependency.acompletion.assert_awaited_once() @pytest.mark.asyncio @pytest.mark.parametrize("failure", [ValueError("invalid_encrypted_content"), TimeoutError("classifier timed out")]) From 81f1778f535d3542e89e2b30eaf61b10a42cb4a8 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 13:05:58 -0700 Subject: [PATCH 339/411] feat(proxy): gate organization endpoints on an enterprise license The Admin UI and the docs already present Organizations as an enterprise feature, but every /organization route served unlicensed proxies. A router level dependency now enforces the license on all of them, and it resolves the auth dependency first so a bad key still gets 401 rather than 403. Claude-Session: https://claude.ai/code/session_01Se8ERtsqMQ3eVzWiLVMyNS --- .../organization_endpoints.py | 18 ++++++- .../test_organization_endpoints.py | 54 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 1e711b036d2..96e946424bd 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -80,7 +80,23 @@ if TYPE_CHECKING: from prisma.models import LiteLLM_OrganizationTable as PrismaOrganizationTable from prisma.models import LiteLLM_UserTable as PrismaUserTable -router: Final = APIRouter() + +async def _enterprise_license_required( + _user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> None: + from litellm.proxy.proxy_server import premium_user + + if not premium_user: + raise HTTPException( + status_code=403, + detail={ + "error": "Organizations are only available for LiteLLM Enterprise users. " + f"{CommonProxyErrors.not_premium_user.value}" + }, + ) + + +router: Final = APIRouter(dependencies=[Depends(_enterprise_license_required)]) class _ObjectPermissionRow(Protocol): diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index 4d13e054e46..328030a17f8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -1226,3 +1226,57 @@ def test_v2_update_organization_is_in_openapi_schema(): v2_path = app.openapi()["paths"]["/v2/organization/{organization_id}"] assert v2_path["patch"]["tags"] == ["organization management"] assert "OrganizationUpdateRequestV2" in json.dumps(v2_path["patch"]["requestBody"]) + + +def _organization_route_targets() -> list[tuple[str, str]]: + from fastapi.routing import APIRoute + + from litellm.proxy.management_endpoints.organization_endpoints import router + + return [ + (method, route.path.replace("{organization_id}", "org-under-test")) + for route in router.routes + if isinstance(route, APIRoute) + for method in sorted(route.methods - {"HEAD", "OPTIONS"}) + ] + + +def _organization_test_client() -> TestClient: + from fastapi import FastAPI + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.management_endpoints.organization_endpoints import router + + app = FastAPI() + app.include_router(router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="sk-test", user_role=LitellmUserRoles.PROXY_ADMIN + ) + return TestClient(app, raise_server_exceptions=False) + + +@pytest.mark.parametrize(("method", "path"), _organization_route_targets()) +def test_organization_routes_are_blocked_without_enterprise_license(monkeypatch, method, path): + """Every /organization route is enterprise-only, even for a proxy admin.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", False, raising=False) + + response = _organization_test_client().request(method, path, json={}) + + assert response.status_code == 403 + assert "Organizations" in response.json()["detail"]["error"] + + +@pytest.mark.parametrize(("method", "path"), _organization_route_targets()) +def test_organization_routes_pass_the_license_gate_with_enterprise_license(monkeypatch, method, path): + """With a license the gate is transparent: whatever fails next, it is not the license check.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", True, raising=False) + monkeypatch.setattr(proxy_server, "prisma_client", None, raising=False) + + response = _organization_test_client().request(method, path, json={}) + + assert "LiteLLM Enterprise" not in response.text From 55ab5ee53d48ef165886de31144b1e172f8208e1 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:08:46 -0700 Subject: [PATCH 340/411] fix(mcp): preserve identity checks across cached OAuth credentials --- AGENTS.md | 2 + litellm/proxy/_experimental/mcp_server/db.py | 34 +++++--- .../mcp_server/oauth2_token_cache.py | 25 ++++-- .../mcp_server/oauth_identity_binding.py | 2 +- .../authz_code_refresher.py | 11 +++ .../outbound_credentials/oauth_token_store.py | 1 + .../per_user_oauth_store.py | 33 ++++++-- .../outbound_credentials/token_cache_codec.py | 36 ++++++-- .../outbound_credentials/v2_token_store.py | 2 + .../test_authz_code_refresher.py | 18 ++-- .../test_per_user_oauth_store.py | 82 +++++++++++++++++-- .../test_token_cache_codec.py | 14 ++++ .../mcp_server/test_db_credentials.py | 75 +++++++++++++++++ .../mcp_server/test_oauth_identity_binding.py | 30 +++++-- 14 files changed, 311 insertions(+), 54 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 41921fdff4d..a1e8f6f618d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1 +1,3 @@ Read @CLAUDE.md for coding guidelines + +Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or lint budgets to satisfy a check diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 08f83aeb756..789b2ffaef4 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -6,6 +6,7 @@ from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, cast +from fastapi import HTTPException from typing_extensions import ReadOnly from litellm._logging import verbose_proxy_logger @@ -1692,13 +1693,18 @@ async def refresh_user_oauth_token( ) return None - binding_proof: Final = await enforce_oauth_identity_binding( - server=server, - token_response=body, - litellm_user_id=user_id, - grant_type="refresh_token", - refresh_ownership=RefreshTokenPresented(refresh_token), - ) + try: + binding_proof: Final = await enforce_oauth_identity_binding( + server=server, + token_response=body, + litellm_user_id=user_id, + grant_type="refresh_token", + refresh_ownership=RefreshTokenPresented(refresh_token), + ) + except HTTPException as exc: + if exc.status_code != 403: + raise + return None access_token: Final[str | None] = body.get("access_token") if not access_token: @@ -1812,8 +1818,14 @@ async def resolve_user_oauth_access_token( binding: Final = server.oauth_identity_binding enforce_binding: Final = binding is not None and binding.mode == "enforce" - if enforce_binding: - await mcp_per_user_token_cache.delete(user_id, server_id) + if prefetched_creds is None and enforce_binding and binding is not None: + bound_token: Final = await mcp_per_user_token_cache.get_token(user_id, server_id) + if bound_token is not None: + if await credential_binding_matches( + binding, user_id, server_id, {"identity_binding_proof": bound_token.identity_binding_proof} + ): + return bound_token.access_token + await mcp_per_user_token_cache.delete(user_id, server_id) if prefetched_creds is None and not enforce_binding: cached_token: Final = await mcp_per_user_token_cache.get(user_id, server_id) if cached_token is not None: @@ -1848,7 +1860,9 @@ async def resolve_user_oauth_access_token( access_token: Final[str] = cred["access_token"] if prefetched_creds is None: ttl: Final = _compute_per_user_token_ttl(server, _remaining_token_seconds(cred.get("expires_at"))) - await mcp_per_user_token_cache.set(user_id, server_id, access_token, ttl) + await mcp_per_user_token_cache.set( + user_id, server_id, access_token, ttl, identity_binding_proof=cred.get("identity_binding_proof") + ) return access_token except Exception as e: verbose_proxy_logger.warning( diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py index a4ef970b87a..42edc2999ab 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py +++ b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py @@ -28,6 +28,8 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import ( build_upstream_oauth2_token_request, resolve_upstream_resource, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import OAuthToken +from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import OAuthTokenCacheCodec from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -233,8 +235,17 @@ class MCPPerUserTokenCache: def _cache_key(self, user_id: str, server_id: str) -> str: return f"{MCP_PER_USER_TOKEN_REDIS_KEY_PREFIX}:{user_id}:{server_id}" + def _codec(self) -> OAuthTokenCacheCodec: + return OAuthTokenCacheCodec( + encrypt_value_helper, + lambda blob: decrypt_value_helper(blob, key="mcp_per_user_token", exception_type="debug"), + ) + async def get(self, user_id: str, server_id: str) -> str | None: - """Return the plaintext access_token, or None on miss/error.""" + token: Final = await self.get_token(user_id, server_id) + return token.access_token if token is not None else None + + async def get_token(self, user_id: str, server_id: str) -> OAuthToken | None: try: from litellm.proxy.proxy_server import user_api_key_cache # noqa: PLC0415 @@ -242,12 +253,7 @@ class MCPPerUserTokenCache: encrypted: Final = await user_api_key_cache.async_get_cache(key) if encrypted is None: return None - plaintext: Final = decrypt_value_helper( - encrypted, - key="mcp_per_user_token", - exception_type="debug", - ) - return plaintext or None + return self._codec().decode(encrypted) except Exception as exc: verbose_logger.debug( "MCPPerUserTokenCache.get failed for user=%s server=%s: %s", @@ -263,13 +269,16 @@ class MCPPerUserTokenCache: server_id: str, access_token: str, ttl: int, + identity_binding_proof: str | None = None, ) -> None: """Store NaCl-encrypted access_token in Redis with the given TTL.""" try: from litellm.proxy.proxy_server import user_api_key_cache # noqa: PLC0415 key: Final = self._cache_key(user_id, server_id) - encrypted: Final = encrypt_value_helper(access_token) + encrypted: Final = self._codec().encode( + OAuthToken(access_token=access_token, identity_binding_proof=identity_binding_proof) + ) await user_api_key_cache.async_set_cache(key, encrypted, ttl=ttl) verbose_logger.debug( "MCPPerUserTokenCache.set: cached token for user=%s server=%s ttl=%ds", diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index 7ca9e6a23df..f02e6c85d9b 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -342,7 +342,7 @@ async def _evaluate_binding( claims: Final = _decode_id_token(id_token, binding, signing_key) if isinstance(claims, _BindingRejection): return claims - if grant_type == "authorization_code": + if grant_type == "authorization_code" and (binding.mode == "enforce" or expected_nonce is not None): nonce: Final = claims.get("nonce") if not expected_nonce or not isinstance(nonce, str) or not hmac.compare_digest(nonce, expected_nonce): return _BindingRejection( diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py index 824459fd9b9..af1e82eab82 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py @@ -14,6 +14,8 @@ import time from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING, Final, Protocol +from fastapi import HTTPException + from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( TokenEndpointAuthConfigError, @@ -93,6 +95,14 @@ class AuthorizationCodeRefresher: self._identity_validator = identity_validator async def refresh(self, user_id: str, server_id: str, token: OAuthToken) -> OAuthToken | None: + try: + return await self._refresh(user_id, server_id, token) + except HTTPException as exc: + if exc.status_code != 403: + raise + return None + + async def _refresh(self, user_id: str, server_id: str, token: OAuthToken) -> OAuthToken | None: if token.refresh_token is None: return None server: Final = self._server_lookup(server_id) @@ -162,4 +172,5 @@ class AuthorizationCodeRefresher: expires_at=self._clock() + expires_in if expires_in is not None else None, refresh_token=new_refresh, scopes=scopes, + identity_binding_proof=binding_proof, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py index e0cd9e8e5ed..0ac4296f498 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py @@ -41,6 +41,7 @@ class OAuthToken: expires_at: float | None = None refresh_token: str | None = None scopes: tuple[str, ...] = () + identity_binding_proof: str | None = None def __repr__(self) -> str: has_refresh: Final = self.refresh_token is not None diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py index efcc613c539..2897c3e8e4a 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py @@ -12,6 +12,7 @@ from __future__ import annotations import asyncio from collections.abc import Callable, Mapping +from functools import partial from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger @@ -39,7 +40,6 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_cod OAuthTokenCacheCodec, ) from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import ( - CredentialReader, V2PerUserTokenStore, ) @@ -146,12 +146,26 @@ def _runtime_backend_and_coordinator() -> tuple[TokenCacheBackend | None, Refres return backend, coordinator, True +async def _read_bound_credential( + server_lookup: ServerLookup, user_id: str, server_id: str +) -> Mapping[str, object] | None: + credential: Final = await _read_credential(user_id, server_id) + server: Final = server_lookup(server_id) + binding: Final = server.oauth_identity_binding if server else None + if credential is not None and binding is not None and binding.mode == "enforce": + if not await credential_binding_matches(binding, user_id, server_id, credential): + return None + return credential + + def _build_per_user_oauth_token_store( server_lookup: ServerLookup, ) -> tuple[CachedOAuthTokenStore, bool]: backend, coordinator, uses_redis = _runtime_backend_and_coordinator() refresher: Final = AuthorizationCodeRefresher(server_lookup, _post_token_endpoint, _persist_credential) - refreshing: Final = RefreshingTokenStore(V2PerUserTokenStore(_read_credential), refresher, coordinator=coordinator) + refreshing: Final = RefreshingTokenStore( + V2PerUserTokenStore(partial(_read_bound_credential, server_lookup)), refresher, coordinator=coordinator + ) return CachedOAuthTokenStore(refreshing, default_ttl_seconds=_DEFAULT_TTL_SECONDS, backend=backend), uses_redis @@ -176,10 +190,8 @@ class LazyPerUserOAuthTokenStore: *, store_builder: StoreBuilder = _build_per_user_oauth_token_store, redis_available: Callable[[], bool] = _redis_cache_is_available, - credential_reader: CredentialReader = _read_credential, ) -> None: self._server_lookup = server_lookup - self._credential_reader = credential_reader self._store_builder = store_builder self._redis_available = redis_available self._store: InvalidatableOAuthTokenStore | None = None @@ -188,13 +200,18 @@ class LazyPerUserOAuthTokenStore: self._local_fetches = 0 async def fetch(self, user_id: str, server_id: str) -> OAuthToken | None: + token: Final = await self._fetch_token(user_id, server_id) server: Final = self._server_lookup(server_id) binding: Final = server.oauth_identity_binding if server else None - if binding is not None and binding.mode == "enforce": - credential: Final = await self._credential_reader(user_id, server_id) - await self.invalidate(user_id, server_id) - if credential is None or not await credential_binding_matches(binding, user_id, server_id, credential): + if token is not None and binding is not None and binding.mode == "enforce": + if not await credential_binding_matches( + binding, user_id, server_id, {"identity_binding_proof": token.identity_binding_proof} + ): + await self.invalidate(user_id, server_id) return None + return token + + async def _fetch_token(self, user_id: str, server_id: str) -> OAuthToken | None: if self._uses_redis: store = self._store if store is not None: diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py index 14cc309b685..56dc3f19cdd 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py @@ -1,24 +1,26 @@ """Serialize + encrypt boundary for caching an OAuth token in a shared (Redis) cache. -A cross-replica cache must serialize the token, and a plaintext bearer in Redis is a leak, so this -encrypts the value (NaCl in production via the injected ``encrypt``, identity in tests). It caches -**only** the ``access_token``: the hot path needs just the bearer, expiry is carried by the cache -entry's TTL (set from the token's ``expires_at`` by the cache), and the long-lived refresh_token stays -in the DB - the refresh path is always a cache miss that re-reads it - so it never reaches Redis. A -decoded token therefore carries only the bearer (``expires_at`` and ``refresh_token`` both None); the -TTL, not the value, bounds its life. An empty/undecryptable blob (e.g. master-key rotation) is a miss. +Shared cache values contain an encrypted access token and optional identity-binding proof. +Refresh tokens remain in the database; cache TTL bounds the access token's lifetime. +Legacy bearer-only entries decode without proof and cannot satisfy identity enforcement. """ from __future__ import annotations +import json from collections.abc import Callable from dataclasses import dataclass from typing import Final +from pydantic import TypeAdapter, ValidationError + from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) +_BOUND_PREFIX: Final = "litellm-bound-oauth-v1:" +_BOUND_PAYLOAD: Final = TypeAdapter(dict[str, str]) + @dataclass(frozen=True, slots=True) class OAuthTokenCacheCodec: @@ -26,10 +28,30 @@ class OAuthTokenCacheCodec: decrypt: Callable[[str], str | None] def encode(self, token: OAuthToken) -> str: + if token.identity_binding_proof is not None: + return self.encrypt( + _BOUND_PREFIX + + json.dumps( + { + "access_token": token.access_token, + "identity_binding_proof": token.identity_binding_proof, + } + ) + ) return self.encrypt(token.access_token) def decode(self, blob: str) -> OAuthToken | None: access_token: Final = self.decrypt(blob) if not access_token: return None + if access_token.startswith(_BOUND_PREFIX): + try: + payload: Final = _BOUND_PAYLOAD.validate_json(access_token[len(_BOUND_PREFIX) :]) + except ValidationError: + return None + bearer: Final = payload.get("access_token") + proof: Final = payload.get("identity_binding_proof") + if not bearer or not proof: + return None + return OAuthToken(access_token=bearer, identity_binding_proof=proof) return OAuthToken(access_token=access_token, refresh_token=None) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py index 4d88ec8b025..0f18931d118 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py @@ -46,11 +46,13 @@ def _to_oauth_token(payload: Mapping[str, object]) -> OAuthToken | None: return None refresh_token: Final = payload.get("refresh_token") expires_at: Final = payload.get("expires_at") + binding_proof: Final = payload.get("identity_binding_proof") return OAuthToken( access_token=access_token, expires_at=_iso_to_epoch(expires_at) if isinstance(expires_at, str) else None, refresh_token=refresh_token if isinstance(refresh_token, str) else None, scopes=_to_scopes(payload.get("scopes")), + identity_binding_proof=binding_proof if isinstance(binding_proof, str) else None, ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py index 16707f6ace8..df068b60338 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py @@ -293,17 +293,18 @@ async def test_refresh_uses_admin_entered_token_url_when_issuer_yield_empties_re @pytest.mark.asyncio async def test_identity_rejection_never_persists_or_returns_refreshed_token(): from unittest.mock import AsyncMock + from fastapi import HTTPException validator = AsyncMock(side_effect=HTTPException(status_code=403, detail="oauth_principal_mismatch")) persist = AsyncMock() refresher = AuthorizationCodeRefresher( - _lookup(_Server()), _endpoint({"access_token": "foreign-token", "id_token": "foreign-identity"}), - persist, identity_validator=validator, + _lookup(_Server()), + _endpoint({"access_token": "foreign-token", "id_token": "foreign-identity"}), + persist, + identity_validator=validator, ) - with pytest.raises(HTTPException) as error: - await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) - assert error.value.status_code == 403 + assert await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) is None persist.assert_not_awaited() @@ -314,10 +315,13 @@ async def test_verified_refresh_preserves_binding_proof_in_storage(): validator = AsyncMock(return_value="verified-binding") persist = AsyncMock() refresher = AuthorizationCodeRefresher( - _lookup(_Server()), _endpoint({"access_token": "new", "refresh_token": "rotated"}), - persist, identity_validator=validator, + _lookup(_Server()), + _endpoint({"access_token": "new", "refresh_token": "rotated"}), + persist, + identity_validator=validator, ) token = await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) assert token.access_token == "new" assert token.refresh_token == "rotated" + assert token.identity_binding_proof == "verified-binding" assert persist.await_args.kwargs["identity_binding_proof"] == "verified-binding" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py index bd190b09c82..e7308884060 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py @@ -254,20 +254,86 @@ async def test_enforcement_invalidates_cached_legacy_credentials_before_use(): from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer server = MCPServer( - server_id="srv", name="srv", transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + server_id="srv", + name="srv", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth_identity_binding=MCPOAuthIdentityBinding( - mode="enforce", issuer="https://idp.example.com", audiences=["client"], + mode="enforce", + issuer="https://idp.example.com", + audiences=["client"], ), ) cached = _RecordingStore("belongs-to-bob") - async def read_legacy(user_id, server_id): - return {"access_token": "belongs-to-bob", "refresh_token": "bobs-refresh"} - store = LazyPerUserOAuthTokenStore( - lambda server_id: server, store_builder=lambda lookup: (cached, False), - redis_available=lambda: False, credential_reader=read_legacy, + lambda server_id: server, + store_builder=lambda lookup: (cached, False), + redis_available=lambda: False, ) assert await store.fetch("alice", "srv") is None - assert cached.calls == [] + assert cached.calls == [("alice", "srv")] assert cached.invalidations == [("alice", "srv")] + + +@pytest.mark.asyncio +async def test_enforced_cache_hit_avoids_credential_read_and_rejects_changed_policy(monkeypatch): + from unittest.mock import AsyncMock + + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import current_binding_proof + from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + oauth_identity_binding={ + "mode": "enforce", + "issuer": "https://idp.example", + "audiences": ["client"], + "caller_field": "user_id", + "principal_claim": "sub", + }, + ) + proof = await current_binding_proof(server.oauth_identity_binding, "alice", "srv") + read = AsyncMock(return_value={"access_token": "alice-token", "identity_binding_proof": proof}) + monkeypatch.setattr(module, "_read_credential", read) + monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False)) + store = LazyPerUserOAuthTokenStore(lambda _: server, redis_available=lambda: False) + assert (await store.fetch("alice", "srv")).access_token == "alice-token" + assert (await store.fetch("alice", "srv")).access_token == "alice-token" + read.assert_awaited_once_with("alice", "srv") + server.oauth_identity_binding = server.oauth_identity_binding.model_copy(update={"audiences": ["changed"]}) + assert await store.fetch("alice", "srv") is None + read.assert_awaited_once() + assert await store.fetch("alice", "srv") is None + assert read.await_count == 2 + + +@pytest.mark.asyncio +async def test_expired_unverified_credential_never_reaches_refresh(monkeypatch): + from unittest.mock import AsyncMock + + from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + token_url="https://idp.example/token", + oauth_identity_binding={"mode": "enforce", "issuer": "https://idp.example", "audiences": ["client"]}, + ) + read = AsyncMock( + return_value={"access_token": "bob", "refresh_token": "bob-refresh", "expires_at": "2000-01-01T00:00:00Z"} + ) + post = AsyncMock() + monkeypatch.setattr(module, "_read_credential", read) + monkeypatch.setattr(module, "_post_token_endpoint", post) + monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False)) + store = LazyPerUserOAuthTokenStore(lambda _: server, redis_available=lambda: False) + assert await store.fetch("alice", "srv") is None + post.assert_not_awaited() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py index 17a7e13af03..12a04c4b4dc 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py @@ -52,3 +52,17 @@ def test_undecryptable_blob_is_a_miss(): def test_empty_plaintext_is_a_miss(): codec = OAuthTokenCacheCodec(encrypt=lambda s: s, decrypt=lambda b: b) assert codec.decode("") is None + + +def test_bound_token_round_trip_preserves_proof_without_refresh_secret(): + codec = _wrapping_codec() + blob = codec.encode(OAuthToken("alice-token", refresh_token="private-refresh", identity_binding_proof="proof")) + assert "private-refresh" not in blob + decoded = codec.decode(blob) + assert decoded == OAuthToken("alice-token", identity_binding_proof="proof") + + +def test_malformed_bound_entries_fail_closed(): + codec = _wrapping_codec() + for payload in ("not-json", "{}", '{"access_token":"at"}', '{"access_token":1,"identity_binding_proof":"p"}'): + assert codec.decode("enc:litellm-bound-oauth-v1:" + payload) is None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 8e2adbeb06e..60a5e1a22bb 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -1629,3 +1629,78 @@ async def test_enforcement_rejects_preexisting_unverified_credential(): cred={"access_token": "belongs-to-bob", "refresh_token": "bobs-refresh-token"}, ) assert result is None + + +@pytest.mark.asyncio +async def test_refresh_identity_rejection_returns_reauthentication_without_persisting(monkeypatch): + from fastapi import HTTPException + from litellm.proxy._experimental.mcp_server import db as module + + validator = AsyncMock(side_effect=HTTPException(status_code=403, detail="oauth_principal_mismatch")) + monkeypatch.setattr(module, "enforce_oauth_identity_binding", validator) + result, captured = await _run_refresh( + monkeypatch, _refresh_server(), {"access_token": "bob", "refresh_token": "rotated"} + ) + assert result is None + assert captured["data"]["grant_type"] == "refresh_token" + module.store_user_oauth_credential.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_verified_legacy_cache_reads_avoid_database_and_reject_policy_changes(monkeypatch): + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import db as module + from litellm.proxy._experimental.mcp_server.oauth2_token_cache import mcp_per_user_token_cache + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import current_binding_proof + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + oauth_identity_binding={ + "mode": "enforce", + "issuer": "https://idp.example", + "audiences": ["client"], + "caller_field": "user_id", + "principal_claim": "sub", + }, + ) + proof = await current_binding_proof(server.oauth_identity_binding, "alice", "srv") + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) + read = AsyncMock(return_value=None) + monkeypatch.setattr(module, "get_user_oauth_credential", read) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + await mcp_per_user_token_cache.set("alice", "srv", "alice-token", 60, identity_binding_proof=proof) + assert await module.resolve_user_oauth_access_token("alice", server) == "alice-token" + assert await module.resolve_user_oauth_access_token("alice", server) == "alice-token" + read.assert_not_awaited() + server.oauth_identity_binding = server.oauth_identity_binding.model_copy(update={"audiences": ["changed"]}) + assert await module.resolve_user_oauth_access_token("alice", server) is None + read.assert_awaited_once() + assert await mcp_per_user_token_cache.get_token("alice", "srv") is None + + +@pytest.mark.asyncio +async def test_unverified_legacy_cache_cannot_bypass_enforcement(monkeypatch): + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import db as module + from litellm.proxy._experimental.mcp_server.oauth2_token_cache import mcp_per_user_token_cache + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + oauth_identity_binding={"mode": "enforce", "issuer": "https://idp.example", "audiences": ["client"]}, + ) + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(module, "get_user_oauth_credential", AsyncMock(return_value={"access_token": "bob"})) + await mcp_per_user_token_cache.set("alice", "srv", "bob", 60) + assert await module.resolve_user_oauth_access_token("alice", server) is None + assert await mcp_per_user_token_cache.get("alice", "srv") is None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py index 6f291865e20..0036035f448 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_oauth_identity_binding.py @@ -13,14 +13,14 @@ from pydantic import ValidationError from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( RefreshOwnershipProven, - VerifiedRefreshToken, RefreshTokenPresented, - current_binding_proof, + VerifiedRefreshToken, _discover_jwks_url, _fetch_issuer_jwks, _load_caller_principal, _load_stored_refresh_token, _select_signing_key, + current_binding_proof, enforce_oauth_identity_binding, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -471,19 +471,20 @@ async def test_refresh_with_mismatched_id_token_rejected(): @pytest.mark.asyncio -async def test_audit_mode_logs_but_does_not_reject(): +async def test_audit_mode_logs_but_does_not_reject(caplog): token: Final = _sign_id_token({"email": "mallory@example.com", "email_verified": True}) result: Final = await enforce_oauth_identity_binding( server=_server(mode="audit"), token_response={"access_token": "at", "id_token": token}, litellm_user_id="user-a", grant_type="authorization_code", - expected_nonce="test-nonce", refresh_ownership=None, jwks_fetcher=_jwks_fetcher, caller_principal_loader=_caller_loader("alice@example.com"), ) assert result is None + assert "oauth_principal_mismatch" in caplog.text + assert "nonce" not in caplog.text @pytest.mark.asyncio @@ -642,6 +643,25 @@ async def test_binding_proof_rejects_changed_user_or_policy(): def test_identity_binding_rejects_modes_without_gateway_credential_custody(auth_type): with pytest.raises(ValidationError, match="gateway-managed per-user"): MCPServer( - server_id="srv", name="srv", transport=MCPTransport.http, auth_type=auth_type, + server_id="srv", + name="srv", + transport=MCPTransport.http, + auth_type=auth_type, oauth_identity_binding=_server().oauth_identity_binding, ) + + +@pytest.mark.asyncio +async def test_audit_matching_login_without_nonce_does_not_report_failure(caplog): + token: Final = _sign_id_token({"email": "alice@example.com", "email_verified": True}) + result: Final = await enforce_oauth_identity_binding( + server=_server(mode="audit"), + token_response={"access_token": "at", "id_token": token}, + litellm_user_id="user-a", + grant_type="authorization_code", + refresh_ownership=None, + jwks_fetcher=_jwks_fetcher, + caller_principal_loader=_caller_loader("alice@example.com"), + ) + assert result is None + assert "oauth_identity_binding audit" not in caplog.text From c2176c786e0fc1fed274ade46bbf901fa33c34a3 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 13:12:04 -0700 Subject: [PATCH 341/411] fix(router): redact cookies and preserve legacy classifier logs --- .../sensitive_data_masker.py | 1 + .../test_classifier_logging.py | 13 +++++ .../test_sensitive_data_masker.py | 9 +++- ui/litellm-dashboard/eslint-suppressions.json | 5 -- ... => LogDetailContent.integration.test.tsx} | 51 +++++++++++++++++++ .../LogDetailsDrawer/LogDetailContent.tsx | 15 +++--- 6 files changed, 80 insertions(+), 14 deletions(-) rename ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/{LogDetailContent.test.tsx => LogDetailContent.integration.test.tsx} (89%) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 22dd4170963..b4c1beea33e 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -15,6 +15,7 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( "token", "auth", "authorization", + "cookie", "credential", # Plural form: Vertex uses ``vertex_credentials``; segment-exact # matching otherwise misses it because "credential" != "credentials". diff --git a/tests/test_litellm/litellm_core_utils/test_classifier_logging.py b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py index c923225c750..ccf14ff06c7 100644 --- a/tests/test_litellm/litellm_core_utils/test_classifier_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_classifier_logging.py @@ -33,6 +33,19 @@ def test_originating_snapshot_masks_nested_credentials_without_altering_source() assert body["metadata"]["nested"][0]["Authorization"] == "Bearer secret" +@pytest.mark.parametrize("header", ["Cookie", "cookie", "COOKIE", "sEt-CoOkIe"]) +def test_originating_snapshot_redacts_cookie_headers_shared_with_caller_metadata(header: str) -> None: + headers: Final = {header: "session=synthetic-session-credential", "content-type": "application/json"} + body: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {"headers": headers}} + snapshot: Final = masked_originating_request({"proxy_server_request": {"body": body, "headers": headers}}) + assert snapshot == { + "messages": [{"role": "user", "content": "hello"}], + "metadata": {"headers": {header: "REDACTED", "content-type": "application/json"}}, + } + assert headers[header] == "session=synthetic-session-credential" + assert body["metadata"]["headers"][header] == "session=synthetic-session-credential" + + @pytest.mark.parametrize("value", [None, "not-json", [], {"messages": object()}]) def test_invalid_provider_payload_is_not_reported_as_captured(value: object) -> None: assert classifier_input_snapshot(value) is None diff --git a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py index c8b6772d965..1551c3fd6e6 100644 --- a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py @@ -358,7 +358,12 @@ def test_redact_credentials_in_payload_leaves_no_fragment_of_the_secret(): "azure_ad_token": fake_token, "aws_secret_access_key": "fake-aws-secret-0000", "vertex_credentials": {"private_key": "fake-pem"}, - "extra_headers": {"Authorization": "Bearer fake-bearer-0000", "x-request-id": "abc123"}, + "extra_headers": { + "Authorization": "Bearer fake-bearer-0000", + "Cookie": "session=fake-session", + "Set-Cookie": "session=fake-session; HttpOnly", + "x-request-id": "abc123", + }, "model": "gpt-4o-mini", "max_tokens": 17, "temperature": 0.25, @@ -373,6 +378,8 @@ def test_redact_credentials_in_payload_leaves_no_fragment_of_the_secret(): assert "fake-bearer-0000" not in str(result) assert result["api_key"] == "REDACTED" assert result["extra_headers"]["Authorization"] == "REDACTED" + assert result["extra_headers"]["Cookie"] == "REDACTED" + assert result["extra_headers"]["Set-Cookie"] == "REDACTED" assert result["extra_headers"]["x-request-id"] == "abc123" assert result["model"] == "gpt-4o-mini" assert result["max_tokens"] == 17 diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 76ac60a6453..e2a08a40bcb 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2325,11 +2325,6 @@ "count": 4 } }, - "src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx": { - "no-nested-ternary": { - "count": 3 - } - }, "src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx": { "no-nested-ternary": { "count": 2 diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx similarity index 89% rename from ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx rename to ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx index e0d6a1d41e0..d538b16d6a2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx @@ -272,6 +272,57 @@ describe("LogDetailContent", () => { expect(screen.queryByRole("tab", { name: "Request" })).not.toBeInTheDocument(); }); + it.each(["object", "serialized", "messages only", "null captures"])( + "preserves request inspection and copying for classifier logs with %s data", + async (shape) => { + const user = userEvent.setup(); + const messages = [{ role: "user", content: "legacy classifier prompt" }]; + const request = { + messages, + temperature: 0.5, + ...(shape === "null captures" ? { classifier_input: null, originating_request_masked: null } : {}), + }; + const storedRequest = shape === "serialized" ? JSON.stringify(request) : request; + const logEntry: Partial = { + call_type: "acompletion", + messages, + proxy_server_request: shape === "messages only" ? undefined : storedRequest, + metadata: { status: "success", internal_call_origin: "autorouter_classifier" }, + }; + render(); + + expect(screen.getByText("legacy classifier prompt")).toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "Classifier input" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("tab", { name: "JSON", exact: true })); + await user.click(screen.getByRole("button", { name: "Copy JSON", exact: true })); + expect(await navigator.clipboard.readText()).toBe( + JSON.stringify(shape === "messages only" ? messages : request, null, 2), + ); + }, + ); + + it.each(["classifier_input", "originating_request_masked"])( + "shows partial classifier audits when only %s is captured", + (field) => { + render( + , + ); + + expect(screen.getByRole("region", { name: "Classifier input" })).toBeInTheDocument(); + expect(screen.getByRole("region", { name: "Originating request, credentials masked" })).toBeInTheDocument(); + expect(screen.getByText("Not captured or message logging disabled")).toBeInTheDocument(); + expect(screen.queryByText("Request & Response")).not.toBeInTheDocument(); + }, + ); + it("should display Request and Response tabs when JSON view is selected", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index e29f2810926..69be5425ea1 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -73,6 +73,9 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok const isClassifier = metadata.internal_call_origin === AUTOROUTER_CLASSIFIER_ORIGIN && (logEntry.call_type === "completion" || logEntry.call_type === "acompletion"); + const rawRequest = formatData(logEntry.proxy_server_request || logEntry.messages); + const hasClassifierAudit = + isClassifier && (rawRequest?.classifier_input != null || rawRequest?.originating_request_masked != null); const hasMessages = checkHasMessages(logEntry.messages); const hasResponse = checkHasResponse(logEntry.response); @@ -93,10 +96,6 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok // Vector store data const hasVectorStoreData = checkHasVectorStoreData(metadata); - const getRawRequest = () => { - return formatData(logEntry.proxy_server_request || logEntry.messages); - }; - const getFormattedResponse = () => { if (hasError && errorInfo) { return { @@ -202,14 +201,14 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok
) : null} - {!isLoadingDetails && isClassifier && ( - + {!isLoadingDetails && hasClassifierAudit && ( + )} - {!isLoadingDetails && !isClassifier && ( + {!isLoadingDetails && !hasClassifierAudit && ( rawRequest} getFormattedResponse={getFormattedResponse} logEntry={logEntry} /> From cc401c40419c71fe1fff24323d6165716f03cf3c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:16:01 -0700 Subject: [PATCH 342/411] fix(ui): skip organization fetches when the session is not premium --- .../organizations/useOrganizations.test.ts | 52 ++++++++++++++++--- .../hooks/organizations/useOrganizations.ts | 9 ++-- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.test.ts index 960afe7392c..5448b7182d7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.test.ts @@ -81,7 +81,7 @@ describe("useOrganizations", () => { userRole: "Admin", token: "test-token", userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -181,7 +181,7 @@ describe("useOrganizations", () => { userRole: "Admin", token: null, userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -197,6 +197,26 @@ describe("useOrganizations", () => { expect(organizationListCall).not.toHaveBeenCalled(); }); + it("does not call the organization API when the session is not premium", async () => { + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userId: "test-user-id", + userRole: "Admin", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useOrganizations(), { wrapper }); + + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + expect(organizationListCall).not.toHaveBeenCalled(); + }); + it("should not execute query when userId is missing", async () => { // Mock missing userId mockUseAuthorized.mockReturnValue({ @@ -205,7 +225,7 @@ describe("useOrganizations", () => { userRole: "Admin", token: "test-token", userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -229,7 +249,7 @@ describe("useOrganizations", () => { userRole: null, token: "test-token", userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -253,7 +273,7 @@ describe("useOrganizations", () => { userRole: null, token: null, userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -335,7 +355,7 @@ describe("useOrganization", () => { userRole: "Admin", token: "test-token", userEmail: "test@example.com", - premiumUser: false, + premiumUser: true, disabledPersonalKeyCreation: null, showSSOBanner: false, }); @@ -356,6 +376,26 @@ describe("useOrganization", () => { expect(result.current.isLoading).toBe(false); }); + it("does not call the organization info API when the session is not premium", () => { + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userId: "test-user-id", + userRole: "Admin", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useOrganization("org-1"), { wrapper }); + + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + expect(organizationInfoCall).not.toHaveBeenCalled(); + }); + it("falls through to the detail API call when no cached list contains the organization", async () => { (organizationInfoCall as any).mockResolvedValue(mockOrganizations[0]); queryClient.setQueryData(organizationKeys.list({ filters: { org_id: "org-2" } }), [mockOrganizations[1]]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts index 734c1986f8f..98053fdf038 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts @@ -11,9 +11,10 @@ export interface OrganizationListFilters { } export const useOrganizations = (filters?: OrganizationListFilters): UseQueryResult => { - const { accessToken, userId, userRole } = useAuthorized(); + const { accessToken, userId, userRole, premiumUser } = useAuthorized(); const orgId = filters?.org_id || null; const orgAlias = filters?.org_alias || null; + const hasSession = Boolean(accessToken && userId && userRole); return useQuery({ queryKey: organizationKeys.list( orgId || orgAlias @@ -21,16 +22,16 @@ export const useOrganizations = (filters?: OrganizationListFilters): UseQueryRes : {}, ), queryFn: async () => await organizationListCall(accessToken!, orgId, orgAlias), - enabled: Boolean(accessToken && userId && userRole), + enabled: hasSession && premiumUser === true, }); }; export const useOrganization = (organizationID?: string) => { const queryClient = useQueryClient(); - const { accessToken } = useAuthorized(); + const { accessToken, premiumUser } = useAuthorized(); return useQuery({ queryKey: organizationKeys.detail(organizationID!), - enabled: Boolean(accessToken && organizationID), + enabled: Boolean(accessToken && organizationID) && premiumUser === true, queryFn: async () => { if (!accessToken || !organizationID) { From b0d66a15b86a5e5d2f503d8c579b5e3a528ab3a0 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 10 Sep 2026 13:18:41 -0700 Subject: [PATCH 343/411] feat(ocr): add core foundation and Mistral adapter (#40530) * feat(ocr): add core foundation and transport primitives * fix(ocr): decline missing Mistral credentials * fix(rust): compile trace parity on Rust 1.98 * refactor(ocr): define native response capability * refactor(auth): generalize missing API key errors * refactor(core): keep URL helpers usage scoped * refactor(ocr): support native responses across adapters * refactor(ocr): preserve unmapped provider params * refactor(ocr): distinguish request preparation from payload transforms * refactor(ocr): trace payload transformation at codec boundary --- litellm-rust/Cargo.lock | 2 + litellm-rust/Cargo.toml | 1 + .../src/audio_transcription/hooks.rs | 2 +- .../crates/ai-gateway/src/ocr/hooks.rs | 2 +- .../ai-gateway/src/routes/messages/mod.rs | 3 +- .../crates/ai-gateway/src/trace_parity.rs | 8 +- litellm-rust/crates/core/Cargo.toml | 8 + litellm-rust/crates/core/src/constants.rs | 3 + litellm-rust/crates/core/src/error.rs | 107 +++++++++ litellm-rust/crates/core/src/http_utils.rs | 105 +++++++- litellm-rust/crates/core/src/lib.rs | 1 + .../crates/core/src/ocr/adapters/mistral.rs | 147 ++++++++++++ .../crates/core/src/ocr/adapters/mod.rs | 69 ++++++ litellm-rust/crates/core/src/ocr/client.rs | 75 ++++++ .../crates/core/src/ocr/codecs/mistral/mod.rs | 5 + .../src/ocr/codecs/mistral/transformation.rs | 102 ++++++++ .../core/src/ocr/codecs/mistral/types.rs | 53 ++++ .../crates/core/src/ocr/codecs/mod.rs | 1 + litellm-rust/crates/core/src/ocr/error.rs | 42 ++++ litellm-rust/crates/core/src/ocr/handler.rs | 72 ++++++ litellm-rust/crates/core/src/ocr/hooks.rs | 146 +++++++++++ litellm-rust/crates/core/src/ocr/mod.rs | 19 ++ litellm-rust/crates/core/src/ocr/prepare.rs | 142 +++++++++++ litellm-rust/crates/core/src/ocr/registry.rs | 53 ++++ .../crates/core/src/ocr/transformation.rs | 6 +- litellm-rust/crates/core/src/ocr/types.rs | 154 ++++++++++-- litellm-rust/crates/core/src/ocr/wire.rs | 154 ++++++++++++ .../providers/azure_ai/ocr/transformation.rs | 14 +- .../providers/mistral/ocr/transformation.rs | 11 +- .../providers/reducto/ocr/transformation.rs | 10 +- .../providers/vertex_ai/ocr/transformation.rs | 8 +- litellm-rust/crates/core/src/url_utils.rs | 95 ++++++++ litellm-rust/crates/core/tests/ocr.rs | 227 ++++++++++++++++++ litellm-rust/crates/core/tests/ocr/support.rs | 106 ++++++++ .../crates/python-bridge/src/errors.rs | 1 + 35 files changed, 1904 insertions(+), 50 deletions(-) create mode 100644 litellm-rust/crates/core/src/ocr/adapters/mistral.rs create mode 100644 litellm-rust/crates/core/src/ocr/adapters/mod.rs create mode 100644 litellm-rust/crates/core/src/ocr/client.rs create mode 100644 litellm-rust/crates/core/src/ocr/codecs/mistral/mod.rs create mode 100644 litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs create mode 100644 litellm-rust/crates/core/src/ocr/codecs/mistral/types.rs create mode 100644 litellm-rust/crates/core/src/ocr/codecs/mod.rs create mode 100644 litellm-rust/crates/core/src/ocr/error.rs create mode 100644 litellm-rust/crates/core/src/ocr/handler.rs create mode 100644 litellm-rust/crates/core/src/ocr/hooks.rs create mode 100644 litellm-rust/crates/core/src/ocr/prepare.rs create mode 100644 litellm-rust/crates/core/src/ocr/registry.rs create mode 100644 litellm-rust/crates/core/src/ocr/wire.rs create mode 100644 litellm-rust/crates/core/src/url_utils.rs create mode 100644 litellm-rust/crates/core/tests/ocr.rs create mode 100644 litellm-rust/crates/core/tests/ocr/support.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 43b9ec1aac2..7addc4e3e45 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1453,11 +1453,13 @@ dependencies = [ "rstest", "serde", "serde_json", + "serde_path_to_error", "sha2 0.10.9", "thiserror 2.0.19", "tokio", "tracing", "tracing-subscriber", + "url", ] [[package]] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 82de7f40069..00fa6faaaef 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -39,6 +39,7 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" +url = "2.5.8" [profile.release] opt-level = 3 diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs index dbe2d3a325b..098ce071efc 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs @@ -269,7 +269,7 @@ fn guardrail_error_to_core_error(error: GuardrailError) -> Error { fn core_error_kind(error: &Error) -> &'static str { match error { - Error::Auth(_) => "AuthError", + Error::Auth(_) | Error::MissingApiKey { .. } => "AuthError", Error::InvalidProvider(_) => "InvalidProvider", Error::InvalidRequest(_) => "InvalidRequest", Error::InvalidType { .. } => "InvalidType", diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs index ed41f1ff9e7..d1dd811f7ab 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs @@ -386,7 +386,7 @@ fn guardrail_error_to_core_error(error: GuardrailError) -> Error { fn core_error_kind(error: &Error) -> &'static str { match error { - Error::Auth(_) => "AuthError", + Error::Auth(_) | Error::MissingApiKey { .. } => "AuthError", Error::InvalidProvider(_) => "InvalidProvider", Error::InvalidRequest(_) => "InvalidRequest", Error::InvalidType { .. } => "InvalidType", diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs index bb9f3851a77..c22d05f5726 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs @@ -114,7 +114,8 @@ impl IntoResponse for MessagesRouteError { | Error::Connect(_) | Error::InvalidResponse(_) | Error::InvalidType { .. } - | Error::MissingField(_) => ( + | Error::MissingField(_) + | Error::MissingApiKey { .. } => ( StatusCode::BAD_GATEWAY, "messages provider request failed".to_string(), ), diff --git a/litellm-rust/crates/ai-gateway/src/trace_parity.rs b/litellm-rust/crates/ai-gateway/src/trace_parity.rs index 614852c541d..21123df3f1c 100644 --- a/litellm-rust/crates/ai-gateway/src/trace_parity.rs +++ b/litellm-rust/crates/ai-gateway/src/trace_parity.rs @@ -47,10 +47,10 @@ pub async fn messages_request( .header(CONTENT_TYPE, "application/json") .body(Body::from(body.to_string())) .map_err(|error| Error::InvalidRequest(error.to_string()))?; - let response = routes::app(state) - .oneshot(request) - .await - .map_err(|error| match error {})?; + let response = match routes::app(state).oneshot(request).await { + Ok(response) => response, + Err(error) => match error {}, + }; let status: StatusCode = response.status(); let bytes = to_bytes(response.into_body(), usize::MAX) .await diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index c0de7ff3977..b4ca88cf16f 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -4,6 +4,11 @@ version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true +autotests = false + +[[test]] +name = "workspace_crate_allowlist" +path = "tests/workspace_crate_allowlist.rs" [dependencies] base64.workspace = true @@ -11,10 +16,13 @@ rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true +serde_path_to_error = "0.1" +tokio.workspace = true thiserror.workspace = true tracing.workspace = true tracing-subscriber = { workspace = true, optional = true } sha2.workspace = true +url.workspace = true aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true } aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index fc81f4fa029..108d2a48e30 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -43,3 +43,6 @@ pub const EMPTY_TEXT_PLACEHOLDER: &str = "[System: Empty message content sanitised to satisfy protocol]"; pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace"; +pub(crate) const OCR_HTTP_TIMEOUT_SECS: u64 = 600; +pub(crate) const OCR_CONNECT_TIMEOUT_SECS: u64 = 10; +pub(crate) const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1"; diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index db3fa2ec704..0382314057f 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -17,6 +17,10 @@ pub enum Error { InvalidRequest(String), #[error("{0}")] Auth(String), + #[error( + "Missing {provider} API Key - A call is being made to {provider} but no key is set either in the environment variables or via params" + )] + MissingApiKey { provider: &'static str }, #[error("upstream request failed with status {status}: {body}")] Http { status: u16, body: String }, #[error("upstream network error: {0}")] @@ -36,6 +40,59 @@ pub enum Error { Unsupported(&'static str), } +#[derive(Clone, Debug, ThisError, PartialEq, Eq)] +pub enum TransportError { + #[error("upstream request failed with status {status}: {body}")] + Http { status: u16, body: String }, + #[error("upstream network error: {0}")] + Network(String), + #[error("could not reach the provider: {0}")] + Connect(String), +} + +impl TransportError { + pub fn from_reqwest_before_dispatch(error: reqwest::Error) -> Self { + let before_dispatch = !error.is_timeout() && (error.is_connect() || error.is_builder()); + let message = error.without_url().to_string(); + if before_dispatch { + Self::Connect(message) + } else { + Self::Network(message) + } + } +} + +impl From for TransportError { + fn from(error: reqwest::Error) -> Self { + Self::Network(error.without_url().to_string()) + } +} + +impl From for Error { + fn from(error: crate::ocr::error::OcrRequestError) -> Self { + match error { + crate::ocr::error::OcrRequestError::MissingField(field) => Self::MissingField(field), + error => Self::InvalidRequest(error.to_string()), + } + } +} + +impl From for Error { + fn from(error: crate::ocr::error::OcrResponseError) -> Self { + Self::InvalidResponse(error.to_string()) + } +} + +impl From for Error { + fn from(error: TransportError) -> Self { + match error { + TransportError::Http { status, body } => Self::Http { status, body }, + TransportError::Network(message) => Self::Network(message), + TransportError::Connect(message) => Self::Connect(message), + } + } +} + pub fn json_type_name(value: &serde_json::Value) -> &'static str { match value { serde_json::Value::Null => "null", @@ -46,3 +103,53 @@ pub fn json_type_name(value: &serde_json::Value) -> &'static str { serde_json::Value::Object(_) => "object", } } + +#[cfg(test)] +mod transport_tests { + use super::*; + + #[tokio::test] + async fn transport_errors_remove_urls_and_keep_dispatch_context() { + let error = reqwest::Client::builder() + .no_proxy() + .build() + .expect("client") + .get("http://localhost:invalid/private?api_key=secret") + .send() + .await + .expect_err("invalid port"); + let error = TransportError::from_reqwest_before_dispatch(error); + assert!(matches!(error, TransportError::Connect(_))); + assert!(!error.to_string().contains("secret")); + assert!(!error.to_string().contains("private")); + } + + #[tokio::test] + async fn request_timeout_is_not_safe_to_retry_as_a_connect_failure() { + use std::time::Duration; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let address = listener.local_addr().expect("address"); + let request = reqwest::Client::builder() + .no_proxy() + .build() + .expect("client") + .get(format!("http://{address}")) + .timeout(Duration::from_millis(200)) + .send(); + let (response, accepted) = tokio::join!( + request, + tokio::time::timeout(Duration::from_secs(2), listener.accept()) + ); + let _connection = accepted + .expect("accept deadline") + .expect("accepted connection"); + let error = response.expect_err("server does not respond"); + assert!(error.is_timeout()); + assert!(matches!( + TransportError::from_reqwest_before_dispatch(error), + TransportError::Network(_) + )); + } +} diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index cb472dd5a57..9299bb77ac8 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -1,10 +1,43 @@ -//! Header and upstream-body helpers shared by every route module. - use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; use crate::error::{Error, json_type_name}; +#[allow( + dead_code, + reason = "used by the OCR architecture in the next stacked PR" +)] +pub(crate) enum HeaderPolicy<'a> { + All, + Only(&'a [&'a str]), + Except(&'a [&'a str]), +} + +#[allow( + dead_code, + reason = "used by the OCR architecture in the next stacked PR" +)] +pub(crate) fn with_headers( + builder: reqwest::RequestBuilder, + headers: &[(String, String)], + policy: HeaderPolicy<'_>, +) -> reqwest::RequestBuilder { + headers + .iter() + .filter(|(name, _)| match policy { + HeaderPolicy::All => true, + HeaderPolicy::Only(names) => names + .iter() + .any(|allowed| name.eq_ignore_ascii_case(allowed)), + HeaderPolicy::Except(names) => !names + .iter() + .any(|excluded| name.eq_ignore_ascii_case(excluded)), + }) + .fold(builder, |builder, (name, value)| { + builder.header(name, value) + }) +} + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn http_request( request: reqwest::RequestBuilder, @@ -12,8 +45,6 @@ pub async fn http_request( request.send().await } -/// Bound an upstream error body before it crosses a host boundary, so provider -/// bodies stay data-minimized. pub fn truncate_error_body(body: &str) -> String { if body.chars().count() <= UPSTREAM_ERROR_BODY_MAX_CHARS { return body.to_string(); @@ -61,11 +92,77 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool { }) } +#[allow( + dead_code, + reason = "used by the OCR architecture in the next stacked PR" +)] +pub(crate) fn deserialize_optional_param<'de, D, T>( + deserializer: D, +) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + T: serde::Deserialize<'de>, +{ + as serde::Deserialize>::deserialize(deserializer).map(Some) +} + #[cfg(test)] mod tests { use super::*; use serde_json::json; + #[rstest::rstest] + #[case(HeaderPolicy::All, true, true)] + #[case(HeaderPolicy::Only(&["authorization"]), true, false)] + #[case(HeaderPolicy::Except(&["authorization"]), false, true)] + fn forwarding_policy_preserves_matching_headers_and_duplicates( + #[case] policy: HeaderPolicy<'_>, + #[case] auth: bool, + #[case] trace: bool, + ) { + let request = with_headers( + reqwest::Client::new().get("https://example.com"), + &[ + ("AuThOrIzAtIoN".into(), "Bearer token".into()), + ("X-Trace".into(), "first".into()), + ("x-trace".into(), "second".into()), + ], + policy, + ) + .build() + .unwrap(); + assert_eq!(request.headers().contains_key("authorization"), auth); + let traces: Vec<_> = request.headers().get_all("x-trace").iter().collect(); + if trace { + assert_eq!(traces, ["first", "second"]); + } else { + assert!(traces.is_empty()); + } + } + + #[test] + fn multipart_policy_leaves_content_headers_to_reqwest() { + let request = with_headers( + reqwest::Client::new() + .post("https://example.com") + .multipart(reqwest::multipart::Form::new().text("file", "abc")), + &[ + ("Content-Type".into(), "application/json".into()), + ("CONTENT-LENGTH".into(), "0".into()), + ], + HeaderPolicy::Except(&["content-type", "content-length"]), + ) + .build() + .unwrap(); + assert!( + request.headers()["content-type"] + .to_str() + .unwrap() + .starts_with("multipart/form-data; boundary=") + ); + assert_ne!(request.headers()["content-length"], "0"); + } + #[test] fn truncate_leaves_short_bodies_untouched() { assert_eq!(truncate_error_body("short"), "short"); diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index b93e084f57e..3a0896a4d5a 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -14,5 +14,6 @@ pub mod realtime; pub mod responses; pub mod router; pub mod routing_utils; +mod url_utils; pub use error::Error; diff --git a/litellm-rust/crates/core/src/ocr/adapters/mistral.rs b/litellm-rust/crates/core/src/ocr/adapters/mistral.rs new file mode 100644 index 00000000000..ea569ffb34f --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/mistral.rs @@ -0,0 +1,147 @@ +use super::OcrAdapter; +use crate::Error; +use crate::constants::MISTRAL_OCR_API_BASE; +use crate::ocr::OcrClient; +use crate::ocr::codecs::mistral::{self, MistralOcrParams, MistralOcrResponse}; +use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; +use crate::ocr::prepare::{ + _prepare_ocr_request, ParsedProviderParams, credential_env, transform_request_body, +}; +use crate::ocr::registry::OcrProvider; +use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection}; +use crate::url_utils::ApiUrl; + +const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY"; + +#[derive(Clone, Debug)] +pub(crate) struct MistralAdapter; + +impl OcrAdapter for MistralAdapter { + type ProviderResponse = MistralOcrResponse; + const PROVIDER: OcrProvider = OcrProvider::Mistral; + + async fn prepare_request( + &self, + request: &LiteLLMOcrRequest, + client: &OcrClient, + ) -> Result { + let ParsedProviderParams { + known: params, + extra_params: _extra_params, + } = _prepare_ocr_request::(request)?; + let headers = validate_environment(&request.connection, &credential_env)?; + let url = get_complete_url(request.connection.api_base.as_deref())?; + let body = + mistral::transform_ocr_request(&request.model, request.document.clone(), ¶ms)?; + transform_request_body(client, request, &url, &headers, body, |_| Ok(())).await + } + + fn transform_ocr_response( + &self, + request: &LiteLLMOcrRequest, + response: Self::ProviderResponse, + ) -> Result { + mistral::transform_ocr_response(&request.model, response) + } +} + +pub(crate) fn get_complete_url(api_base: Option<&str>) -> Result { + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(MISTRAL_OCR_API_BASE); + ApiUrl::parse(base) + .and_then(|url| url.complete_path(&["v1", "ocr"])) + .map(|url| url.into_string()) + .map_err(|_| { + OcrRequestError::RequestField { + path: "api_base".into(), + } + .into() + }) +} + +fn validate_environment( + connection: &OcrConnection, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, OcrError> { + if crate::http_utils::has_header(&connection.extra_headers, "authorization") { + return Ok(connection.extra_headers.clone()); + } + let api_key = connection + .api_key + .as_deref() + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_string) + .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) + .ok_or(Error::MissingApiKey { + provider: "Mistral", + })?; + Ok( + std::iter::once(("Authorization".into(), format!("Bearer {api_key}"))) + .chain(connection.extra_headers.clone()) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn complete_url_defaults_and_dedupes_v1() { + assert_eq!( + get_complete_url(None).unwrap(), + "https://api.mistral.ai/v1/ocr" + ); + assert_eq!( + get_complete_url(Some("https://example.com/v1?tenant=a")).unwrap(), + "https://example.com/v1/ocr?tenant=a" + ); + assert_eq!( + get_complete_url(Some("https://example.com/v1/ocr?tenant=a")).unwrap(), + "https://example.com/v1/ocr?tenant=a" + ); + } + + #[test] + fn environment_prefers_explicit_key_then_environment() { + let explicit = OcrConnection { + api_key: Some("explicit".into()), + ..OcrConnection::default() + }; + assert_eq!( + validate_environment(&explicit, &|_| Some("environment".into())).unwrap()[0], + ("Authorization".into(), "Bearer explicit".into()) + ); + + assert_eq!( + validate_environment(&OcrConnection::default(), &|_| Some("environment".into())) + .unwrap()[0], + ("Authorization".into(), "Bearer environment".into()) + ); + } + + #[test] + fn environment_preserves_forwarded_authorization() { + let connection = OcrConnection { + extra_headers: vec![("authorization".into(), "Bearer forwarded".into())], + ..OcrConnection::default() + }; + assert_eq!( + validate_environment(&connection, &|_| None).unwrap(), + connection.extra_headers + ); + } + + #[test] + fn environment_rejects_missing_key() { + assert!(matches!( + validate_environment(&OcrConnection::default(), &|_| None), + Err(OcrError::Public(Error::MissingApiKey { + provider: "Mistral" + })) + )); + } +} diff --git a/litellm-rust/crates/core/src/ocr/adapters/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/mod.rs new file mode 100644 index 00000000000..7dfb08b4dc2 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/adapters/mod.rs @@ -0,0 +1,69 @@ +use std::future::Future; + +use serde::de::DeserializeOwned; + +use super::OcrClient; +use super::error::{OcrError, OcrResponseError}; +use super::registry::OcrProvider; +use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrResponseFormat}; +use super::wire::DecodedOcrResponse; + +mod mistral; + +pub(crate) use mistral::MistralAdapter; + +/// Converts a complete LiteLLM OCR call to provider HTTP and normalizes its response. +pub(crate) trait OcrAdapter: Send + Sync + Sized + 'static { + /// Provider JSON schema; direct and Vertex Mistral share `MistralOcrResponse`. + type ProviderResponse: DeserializeOwned + Send; + + const PROVIDER: OcrProvider; + + /// Prepares the complete provider HTTP request. + /// `request` contains the model, document, connection, and unmapped caller options. + /// `client` supplies reusable provider and document HTTP clients. + /// Returns the complete HTTP request, whereas Python returns body data. + fn prepare_request( + &self, + request: &LiteLLMOcrRequest, + client: &OcrClient, + ) -> impl Future> + Send; + + /// Python: `transform_ocr_response`. + /// `request` supplies caller context, including the fallback model. + /// `response` is the decoded provider payload; the output is the shared LiteLLM schema. + fn transform_ocr_response( + &self, + request: &LiteLLMOcrRequest, + response: Self::ProviderResponse, + ) -> Result; + + /// Decodes provider HTTP; adapters may override this to poll asynchronous operations. + /// Python performs that polling inside `async_transform_ocr_response`. + /// `client` is reused for polling; `response` is the initial HTTP response. + /// `url` and `headers` describe the submitted call; `request` supplies limits and format. + fn read_response( + &self, + _client: &OcrClient, + response: reqwest::Response, + _url: &str, + _headers: &[(String, String)], + request: &LiteLLMOcrRequest, + ) -> impl Future, OcrError>> + Send + { + let retain_native = request + .response_format() + .map(|format| format == OcrResponseFormat::Native); + async move { super::client::read_json_response(response, retain_native?).await } + } +} + +macro_rules! for_each_ocr_adapter { + ($callback:ident) => { + $callback! { + Mistral, $crate::ocr::adapters::MistralAdapter, $crate::ocr::adapters::MistralAdapter, Mistral; + } + }; +} + +pub(crate) use for_each_ocr_adapter; diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs new file mode 100644 index 00000000000..36bfc036bae --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -0,0 +1,75 @@ +use std::sync::OnceLock; +use std::time::Duration; + +use serde::de::DeserializeOwned; + +use super::error::OcrError; +use super::handler::perform_ocr_request; +use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; +use super::wire::{DecodedOcrResponse, decode_response}; +use crate::Error; +use crate::constants::OCR_CONNECT_TIMEOUT_SECS; +use crate::error::TransportError; + +#[derive(Clone)] +pub struct OcrClient { + provider_http: reqwest::Client, +} + +impl OcrClient { + pub fn new(provider_http: reqwest::Client) -> Result { + Ok(Self { provider_http }) + } + + #[tracing::instrument( + name = "ocr", + target = "litellm::function_trace", + level = "trace", + skip_all + )] + pub async fn perform(&self, request: LiteLLMOcrRequest) -> Result { + perform_ocr_request(self, request).await + } + + pub(crate) fn provider_http(&self) -> &reqwest::Client { + &self.provider_http + } + + #[cfg(test)] + pub(crate) fn for_test(provider_http: reqwest::Client) -> Self { + Self { provider_http } + } +} + +pub async fn ocr(request: LiteLLMOcrRequest) -> Result { + static CLIENT: OnceLock> = OnceLock::new(); + let client = CLIENT + .get_or_init(|| { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) + .build() + .map_err(TransportError::from) + .and_then(OcrClient::new) + }) + .clone()?; + client.perform(request).await +} + +pub async fn read_json_response( + response: reqwest::Response, + native: bool, +) -> Result, OcrError> { + let status = response.status(); + let bytes = response + .bytes() + .await + .map_err(crate::error::TransportError::from)?; + if !status.is_success() { + return Err(crate::error::TransportError::Http { + status: status.as_u16(), + body: crate::http_utils::truncate_error_body(&String::from_utf8_lossy(&bytes)), + } + .into()); + } + Ok(decode_response(&bytes, native)?) +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/mistral/mod.rs b/litellm-rust/crates/core/src/ocr/codecs/mistral/mod.rs new file mode 100644 index 00000000000..eea4254779e --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/mistral/mod.rs @@ -0,0 +1,5 @@ +mod transformation; +mod types; + +pub(crate) use transformation::{transform_ocr_request, transform_ocr_response}; +pub(crate) use types::{MistralOcrParams, MistralOcrRequest, MistralOcrResponse}; diff --git a/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs new file mode 100644 index 00000000000..cd0a1dc6b17 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs @@ -0,0 +1,102 @@ +use super::{MistralOcrParams, MistralOcrRequest, MistralOcrResponse}; +use crate::ocr::error::{OcrRequestError, OcrResponseError}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub(crate) fn transform_ocr_request( + model: &str, + document: OcrDocument, + params: &MistralOcrParams, +) -> Result { + Ok(MistralOcrRequest { + model: model.to_string(), + document, + params: params.clone(), + }) +} + +pub(crate) fn transform_ocr_response( + model: &str, + response: MistralOcrResponse, +) -> Result { + Ok(LiteLLMOcrResponse { + pages: response.pages, + model: response.model.unwrap_or_else(|| model.to_string()), + document_annotation: response.document_annotation, + usage_info: response.usage_info, + object: "ocr".to_string(), + extra_fields: response.extra_fields, + provider_native_response: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + use serde_json::{Value, json}; + + #[rstest] + #[case("pages", json!([0, 2]))] + #[case("include_image_base64", json!(true))] + #[case("image_limit", json!(2))] + #[case("image_min_size", json!(100))] + #[case("bbox_annotation_format", json!({"type":"json_schema"}))] + #[case("document_annotation_format", json!({"type":"json_schema"}))] + #[case("document_annotation_prompt", json!("extract"))] + #[case("extract_header", json!(true))] + #[case("extract_footer", json!(false))] + #[case("table_format", json!("html"))] + #[case("confidence_scores_granularity", json!("word"))] + #[case("include_blocks", json!(true))] + #[case("id", json!("req-123"))] + fn request_mapping_matches_python(#[case] name: &str, #[case] value: Value) { + let params: MistralOcrParams = + serde_json::from_value(json!({name: value.clone()})).unwrap(); + let document: OcrDocument = serde_json::from_value( + json!({"type":"document_url","document_url":"https://example.com/a.pdf"}), + ) + .unwrap(); + let result = + serde_json::to_value(transform_ocr_request("model", document, ¶ms).unwrap()) + .unwrap(); + assert_eq!(result["model"], "model"); + assert_eq!(result[name], value); + } + + #[test] + fn request_mapping_filters_unknown_fields() { + let params: MistralOcrParams = serde_json::from_value(json!({"unknown": true})).unwrap(); + let document: OcrDocument = serde_json::from_value( + json!({"type":"document_url","document_url":"https://example.com/a.pdf"}), + ) + .unwrap(); + let result = + serde_json::to_value(transform_ocr_request("model", document, ¶ms).unwrap()) + .unwrap(); + assert!(result.get("unknown").is_none()); + } + + #[test] + fn response_preserves_provider_fields() { + let response: MistralOcrResponse = serde_json::from_value(json!({ + "pages":[{"index":0,"markdown":"hello","header":"head","confidence_scores":{"mean":0.99}}], + "model":"returned-model", + "usage_info":{"pages_processed":1,"future_counter":5}, + "future_response_field":"kept" + })) + .unwrap(); + let result = transform_ocr_response("model", response) + .unwrap() + .into_json(); + assert_eq!(result["pages"][0]["header"], "head"); + assert_eq!(result["usage_info"]["future_counter"], 5); + assert_eq!(result["future_response_field"], "kept"); + assert_eq!(result["model"], "returned-model"); + } + + #[test] + fn response_rejects_null_pages() { + assert!(serde_json::from_value::(json!({"pages":null})).is_err()); + } +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/mistral/types.rs b/litellm-rust/crates/core/src/ocr/codecs/mistral/types.rs new file mode 100644 index 00000000000..0e601cd8319 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/mistral/types.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::ocr::types::OcrDocument; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub(crate) struct MistralOcrParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub pages: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub include_image_base64: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub image_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub image_min_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bbox_annotation_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document_annotation_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document_annotation_prompt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extract_header: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extract_footer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub table_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub confidence_scores_granularity: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub include_blocks: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct MistralOcrRequest { + pub model: String, + pub document: OcrDocument, + #[serde(flatten)] + pub params: MistralOcrParams, +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub(crate) struct MistralOcrResponse { + #[serde(default)] + pub pages: Vec, + pub model: Option, + pub document_annotation: Option, + pub usage_info: Option, + #[serde(flatten)] + pub extra_fields: Map, +} diff --git a/litellm-rust/crates/core/src/ocr/codecs/mod.rs b/litellm-rust/crates/core/src/ocr/codecs/mod.rs new file mode 100644 index 00000000000..170ef5f68a7 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/codecs/mod.rs @@ -0,0 +1 @@ +pub(crate) mod mistral; diff --git a/litellm-rust/crates/core/src/ocr/error.rs b/litellm-rust/crates/core/src/ocr/error.rs new file mode 100644 index 00000000000..50793aad566 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/error.rs @@ -0,0 +1,42 @@ +use thiserror::Error; + +use crate::error::TransportError; + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum OcrRequestError { + #[error("Invalid `req_format`. Expected 'native' or 'litellm'.")] + RequestFormat, + #[error("invalid OCR request field: {path}")] + RequestField { path: String }, + #[error("missing required field: {0}")] + MissingField(&'static str), +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum OcrResponseError { + #[error("invalid OCR response field: {path}")] + ResponseField { path: String }, +} + +#[derive(Debug, Error)] +pub enum OcrError { + #[error("{0}")] + Request(#[from] OcrRequestError), + #[error("{0}")] + Response(#[from] OcrResponseError), + #[error("{0}")] + Transport(#[from] TransportError), + #[error("{0}")] + Public(#[from] crate::Error), +} + +impl From for crate::Error { + fn from(error: OcrError) -> Self { + match error { + OcrError::Request(error) => error.into(), + OcrError::Response(error) => error.into(), + OcrError::Transport(error) => error.into(), + OcrError::Public(error) => error, + } + } +} diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs new file mode 100644 index 00000000000..0b04319d966 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -0,0 +1,72 @@ +use super::OcrClient; +use super::adapters::OcrAdapter; +use super::hooks::OcrLifecycleHooks; +use super::registry::OcrAdapterKind; +use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; +use crate::Error; +use crate::call_lifecycle::{CallLifecycle, CallLifecycleContext}; + +pub(crate) async fn perform_ocr_request( + client: &OcrClient, + request: LiteLLMOcrRequest, +) -> Result { + let context = CallLifecycleContext::new( + "ocr", + request.model.clone(), + request.adapter.provider().as_str(), + request + .litellm_call_id + .clone() + .unwrap_or_else(|| format!("ocr-{:032x}", rand::random::())), + ); + let hooks = OcrLifecycleHooks { + hooks: request.hooks.clone(), + provider_name: context.custom_llm_provider.clone(), + }; + CallLifecycle::default().run(context, request, &hooks, |request| async move { + macro_rules! execute_selected_adapter { + ($( $variant:ident, $adapter:ty, $instance:expr, $provider:ident; )+) => { + match request.adapter { + $( OcrAdapterKind::$variant => execute_ocr_provider_call(client, &$instance, request).await, )+ + } + }; + } + super::adapters::for_each_ocr_adapter!(execute_selected_adapter) + }).await +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +async fn execute_ocr_provider_call( + client: &OcrClient, + adapter: &A, + request: LiteLLMOcrRequest, +) -> Result { + let provider_request = adapter.prepare_request(&request, client).await?; + let url = provider_request.url().to_string(); + let headers = provider_request + .headers() + .iter() + .map(|(name, value)| { + value + .to_str() + .map(|value| (name.to_string(), value.to_string())) + .map_err(|_| super::error::OcrRequestError::RequestField { + path: "headers".into(), + }) + }) + .collect::, _>>()?; + let response = crate::http_utils::http_request(reqwest::RequestBuilder::from_parts( + client.provider_http().clone(), + provider_request, + )) + .await + .map_err(crate::error::TransportError::from)?; + let decoded = adapter + .read_response(client, response, &url, &headers, &request) + .await?; + let response = adapter.transform_ocr_response(&request, decoded.data)?; + Ok(LiteLLMOcrResponse { + provider_native_response: decoded.native, + ..response + }) +} diff --git a/litellm-rust/crates/core/src/ocr/hooks.rs b/litellm-rust/crates/core/src/ocr/hooks.rs new file mode 100644 index 00000000000..7dd3c6bf8b2 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/hooks.rs @@ -0,0 +1,146 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrDocument}; +use crate::Error; +use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; +use serde::Serialize; +use serde_json::Value; + +pub type OcrHookFuture<'a, T> = Pin> + Send + 'a>>; +pub type OcrLogFuture<'a> = Pin + Send + 'a>>; + +#[derive(Clone, Debug, Serialize)] +pub struct OcrPreCallRequest { + pub model: String, + pub custom_llm_provider: String, + pub document: OcrDocument, + pub optional_params: Value, +} + +#[derive(Clone, Debug, Serialize)] +pub struct OcrDuringCallRequest { + pub model: String, + pub custom_llm_provider: String, + pub url: String, + pub body: Value, +} + +pub trait OcrHooks: Send + Sync { + fn has_guardrails(&self) -> bool { + false + } + fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> { + Box::pin(async move { Ok(request) }) + } + fn during_call( + &self, + request: OcrDuringCallRequest, + ) -> OcrHookFuture<'_, OcrDuringCallRequest> { + Box::pin(async move { Ok(request) }) + } + fn success<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _response: &'a LiteLLMOcrResponse, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async {}) + } + fn failure<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _error: &'a Error, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async {}) + } +} + +pub struct NoopOcrHooks; +impl OcrHooks for NoopOcrHooks {} + +pub(crate) struct OcrLifecycleHooks { + pub hooks: Arc, + pub provider_name: String, +} + +impl CallLifecycleHooks + for OcrLifecycleHooks +{ + type PreCallFuture<'a> = OcrHookFuture<'a, LiteLLMOcrRequest>; + type DuringCallFuture<'a> = OcrHookFuture<'a, LiteLLMOcrRequest>; + type SuccessFuture<'a> = OcrLogFuture<'a>; + type FailureFuture<'a> = OcrLogFuture<'a>; + + fn async_pre_call_hook<'a>( + &'a self, + _context: &'a CallLifecycleContext, + request: LiteLLMOcrRequest, + ) -> Self::PreCallFuture<'a> { + Box::pin(async move { + if !self.hooks.has_guardrails() { + return Ok(request); + } + let changed = self + .hooks + .pre_call(OcrPreCallRequest { + model: request.model.clone(), + custom_llm_provider: self.provider_name.clone(), + document: request.document, + optional_params: Value::Object(request.optional_params), + }) + .await?; + let Value::Object(optional_params) = changed.optional_params else { + return Err(super::error::OcrRequestError::RequestField { + path: "guardrail.optional_params".into(), + } + .into()); + }; + Ok(LiteLLMOcrRequest { + document: changed.document, + optional_params, + ..request + }) + }) + } + + fn async_during_call_hook<'a>( + &'a self, + _context: &'a CallLifecycleContext, + request: LiteLLMOcrRequest, + ) -> Self::DuringCallFuture<'a> { + Box::pin(async move { Ok(request) }) + } + + #[tracing::instrument( + name = "success_callback", + target = "litellm::function_trace", + level = "trace", + skip_all + )] + fn async_log_success_event<'a>( + &'a self, + context: &'a CallLifecycleContext, + response: &'a LiteLLMOcrResponse, + timing: &'a CallLifecycleTiming, + ) -> Self::SuccessFuture<'a> { + self.hooks.success(context, response, timing) + } + + #[tracing::instrument( + name = "failure_callback", + target = "litellm::function_trace", + level = "trace", + skip_all + )] + fn async_log_failure_event<'a>( + &'a self, + context: &'a CallLifecycleContext, + error: &'a Error, + timing: &'a CallLifecycleTiming, + ) -> Self::FailureFuture<'a> { + self.hooks.failure(context, error, timing) + } +} diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index ec2fbb969a6..78c86aeaa0e 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -1,2 +1,21 @@ +mod adapters; +pub mod client; +mod codecs; +pub mod error; +mod handler; +pub mod hooks; +mod prepare; +mod registry; pub mod transformation; pub mod types; +pub mod wire; + +pub use client::{OcrClient, ocr}; +pub use types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrDocument}; + +#[cfg(test)] +#[path = "../../tests/ocr/support.rs"] +pub(crate) mod test_support; +#[cfg(test)] +#[path = "../../tests/ocr.rs"] +pub(crate) mod tests; diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs new file mode 100644 index 00000000000..cff40b7de50 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -0,0 +1,142 @@ +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::{Map, Value}; + +use super::OcrClient; +use super::error::{OcrError, OcrRequestError}; +use super::hooks::OcrDuringCallRequest; +use super::types::LiteLLMOcrRequest; + +#[derive(Debug, Deserialize)] +pub(crate) struct ParsedProviderParams { + #[serde(flatten)] + pub known: T, + #[serde(default, flatten)] + pub extra_params: Map, +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub(crate) fn _prepare_ocr_request( + request: &LiteLLMOcrRequest, +) -> Result, OcrRequestError> { + super::wire::decode_request_value( + Value::Object(request.optional_params.clone()), + "optional_params", + ) +} + +pub(crate) async fn transform_request_body( + client: &OcrClient, + request: &LiteLLMOcrRequest, + url: &str, + headers: &[(String, String)], + body: B, + validate: impl FnOnce(&B) -> Result<(), OcrRequestError>, +) -> Result +where + B: Serialize + DeserializeOwned, +{ + let body = if request.hooks.has_guardrails() { + let changed = request + .hooks + .during_call(OcrDuringCallRequest { + model: request.model.clone(), + custom_llm_provider: request.adapter.provider().as_str().into(), + url: url.into(), + body: serde_json::to_value(body).map_err(|_| OcrRequestError::RequestField { + path: "body".into(), + })?, + }) + .await?; + let body = OcrWireBody::::decode(changed.body)?; + validate(&body.body)?; + body + } else { + OcrWireBody { + body, + extra: Map::new(), + } + }; + build_http_request(client, request, url, headers, &body) +} + +pub(crate) fn build_http_request( + client: &OcrClient, + request: &LiteLLMOcrRequest, + url: &str, + headers: &[(String, String)], + body: &B, +) -> Result { + let builder = client + .provider_http() + .post(url) + .json(body) + .timeout(request.connection.timeout); + crate::http_utils::with_headers(builder, headers, crate::http_utils::HeaderPolicy::All) + .build() + .map_err(crate::error::TransportError::from) + .map_err(OcrError::from) +} + +#[derive(Serialize)] +struct OcrWireBody { + #[serde(flatten)] + body: B, + #[serde(flatten)] + extra: Map, +} + +impl OcrWireBody { + fn decode(value: Value) -> Result { + let body: B = super::wire::decode_request_value(value.clone(), "guardrail.body")?; + let Value::Object(fields) = value else { + return Err(OcrRequestError::RequestField { + path: "guardrail.body".into(), + }); + }; + let known = serde_json::to_value(&body).map_err(|_| OcrRequestError::RequestField { + path: "guardrail.body".into(), + })?; + let extra = fields + .into_iter() + .filter(|(key, _)| known.get(key).is_none()) + .collect(); + Ok(Self { body, extra }) + } +} + +pub(crate) fn credential_env(name: &str) -> Option { + std::env::var(name).ok() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[derive(Debug, Deserialize, PartialEq)] + struct KnownParams { + pages: Option>, + } + + #[test] + fn parsed_provider_params_separates_known_and_extra_params() { + let parsed: ParsedProviderParams = super::super::wire::decode_request_value( + json!({ + "pages": [0, 2], + "future_ocr_option": true, + "extra_body": {"provider_option": "value"} + }), + "optional_params", + ) + .unwrap(); + + assert_eq!(parsed.known.pages, Some(vec![0, 2])); + assert_eq!(parsed.extra_params["future_ocr_option"], true); + assert_eq!( + parsed.extra_params["extra_body"], + json!({"provider_option": "value"}) + ); + assert_eq!(parsed.extra_params.len(), 2); + } +} diff --git a/litellm-rust/crates/core/src/ocr/registry.rs b/litellm-rust/crates/core/src/ocr/registry.rs new file mode 100644 index 00000000000..9bae8153ee8 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/registry.rs @@ -0,0 +1,53 @@ +use super::adapters::OcrAdapter; +use crate::Error; +use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; + +macro_rules! define_adapter_types { + ($( $variant:ident, $adapter:ty, $instance:expr, $provider:ident; )+) => { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(crate) enum OcrAdapterKind { + $( $variant, )+ + } + + impl OcrAdapterKind { + pub(crate) const fn provider(self) -> OcrProvider { + match self { + $( Self::$variant => <$adapter>::PROVIDER, )+ + } + } + } + }; +} + +super::adapters::for_each_ocr_adapter!(define_adapter_types); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum OcrProvider { + Mistral, +} + +impl OcrProvider { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Mistral => "mistral", + } + } +} + +pub(crate) fn resolve_wire_adapter( + model: &str, + custom_llm_provider: Option<&str>, +) -> Result<(String, OcrAdapterKind), Error> { + let provider = + get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider { + model, + custom_llm_provider: OcrProvider::Mistral.as_str(), + }); + let typed_provider = match provider.custom_llm_provider { + "mistral" => OcrProvider::Mistral, + value => return Err(Error::InvalidProvider(value.to_string())), + }; + match typed_provider { + OcrProvider::Mistral => Ok((provider.model.to_string(), OcrAdapterKind::Mistral)), + } +} diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs index 62299faf9ed..ac4f10bf15b 100644 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -1,7 +1,7 @@ use crate::Error; use serde_json::{Map, Value}; -use super::types::{OcrRequestData, OcrResponseData}; +use super::types::{LiteLLMOcrResponse, OcrRequestData}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum OcrAuthStrategy { @@ -49,14 +49,14 @@ pub trait OcrProviderConfig: Sync { &self, model: &str, response_json: Value, - ) -> Result; + ) -> Result; fn transform_ocr_response_with_params( &self, model: &str, response_json: Value, _optional_params: &Map, - ) -> Result { + ) -> Result { self.transform_ocr_response(model, response_json) } diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index 71cdb232a87..02b5330188c 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -1,6 +1,14 @@ +use std::sync::Arc; +use std::time::Duration; + use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use super::hooks::{NoopOcrHooks, OcrHooks}; +use super::registry::{OcrAdapterKind, resolve_wire_adapter}; +use crate::Error; +use crate::constants::OCR_HTTP_TIMEOUT_SECS; + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct OcrRequestData { pub data: Value, @@ -8,31 +16,145 @@ pub struct OcrRequestData { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct OcrResponseData { +#[serde(tag = "type")] +pub enum OcrDocument { + #[serde(rename = "document_url")] + DocumentUrl { + document_url: String, + #[serde(flatten)] + extra_fields: Map, + }, + #[serde(rename = "image_url")] + ImageUrl { + image_url: String, + #[serde(flatten)] + extra_fields: Map, + }, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum OcrResponseFormat { + #[default] + Litellm, + Native, +} + +#[derive(Clone)] +pub struct OcrConnection { + pub api_key: Option, + pub api_base: Option, + pub extra_headers: Vec<(String, String)>, + pub timeout: Duration, +} + +impl Default for OcrConnection { + fn default() -> Self { + Self { + api_key: None, + api_base: None, + extra_headers: Vec::new(), + timeout: Duration::from_secs(OCR_HTTP_TIMEOUT_SECS), + } + } +} + +pub struct LiteLLMOcrRequest { + pub model: String, + pub document: OcrDocument, + pub connection: OcrConnection, + pub hooks: Arc, + pub litellm_call_id: Option, + pub optional_params: Map, + pub(crate) adapter: OcrAdapterKind, +} + +impl LiteLLMOcrRequest { + pub fn new( + model: String, + document: OcrDocument, + custom_llm_provider: Option<&str>, + optional_params: Map, + ) -> Result { + let (model, adapter_kind) = resolve_wire_adapter(&model, custom_llm_provider)?; + + Ok(Self { + model, + document, + connection: OcrConnection::default(), + hooks: Arc::new(NoopOcrHooks), + litellm_call_id: None, + optional_params, + adapter: adapter_kind, + }) + } + + pub(crate) fn response_format( + &self, + ) -> Result { + self.optional_params + .get("req_format") + .map(|value| { + serde_json::from_value(value.clone()) + .map_err(|_| super::error::OcrRequestError::RequestFormat) + }) + .transpose() + .map(|format| format.unwrap_or_default()) + } + + pub fn with_host_hooks( + self, + hooks: Arc, + litellm_call_id: Option, + ) -> Self { + Self { + hooks, + litellm_call_id, + ..self + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct LiteLLMOcrResponse { pub pages: Vec, pub model: String, pub document_annotation: Option, pub usage_info: Option, pub object: String, + #[serde(flatten)] pub extra_fields: Map, + #[serde(skip_serializing_if = "Option::is_none")] pub provider_native_response: Option, } -impl OcrResponseData { +impl LiteLLMOcrResponse { pub fn into_json(self) -> Value { - let mut response = serde_json::json!({ - "pages": self.pages, - "model": self.model, - "document_annotation": self.document_annotation, - "usage_info": self.usage_info, - "object": self.object, - }); - if let Value::Object(object) = &mut response { - object.extend(self.extra_fields); - if let Some(native_response) = self.provider_native_response { - object.insert("provider_native_response".to_string(), native_response); - } - } - response + serde_json::to_value(self).expect("OCR response fields are JSON-compatible") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn response_serialization_flattens_extra_fields_and_omits_absent_native_response() { + let response = LiteLLMOcrResponse { + pages: vec![], + model: "model".into(), + document_annotation: None, + usage_info: None, + object: "ocr".into(), + extra_fields: json!({"provider_field":"kept"}) + .as_object() + .unwrap() + .clone(), + provider_native_response: None, + }; + let serialized = response.into_json(); + assert_eq!(serialized["provider_field"], "kept"); + assert!(serialized.get("provider_native_response").is_none()); } } diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs new file mode 100644 index 00000000000..f37c06a01ac --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -0,0 +1,154 @@ +use crate::ocr::error::OcrRequestError; +use crate::ocr::error::OcrResponseError; +use std::time::Duration; + +use super::hooks::{OcrDuringCallRequest, OcrPreCallRequest}; +use super::types::{LiteLLMOcrRequest, OcrConnection, OcrDocument}; +use crate::Error; +use serde::{ + Deserialize, + de::{DeserializeOwned, IntoDeserializer}, +}; +use serde_json::{Map, Value}; + +#[derive(Debug)] +pub struct DecodedOcrResponse { + pub data: T, + pub native: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OcrWireRequest { + pub model: String, + pub document: Value, + pub api_key: Option, + pub api_base: Option, + pub custom_llm_provider: Option, + pub extra_headers: Option>, + #[serde(default)] + pub optional_params: Map, + pub timeout_seconds: Option, +} + +pub fn is_supported_request(model: &str, custom_llm_provider: Option<&str>) -> bool { + super::registry::resolve_wire_adapter(model, custom_llm_provider).is_ok() +} + +pub fn decode_request(wire: OcrWireRequest) -> Result { + let document = decode_request_value(wire.document, "document")?; + let headers = wire + .extra_headers + .unwrap_or_default() + .into_iter() + .map(|(name, value)| { + let value = value + .as_str() + .ok_or_else(|| OcrRequestError::RequestField { + path: format!("extra_headers.{name}"), + })?; + Ok((name, value.to_string())) + }) + .collect::, OcrRequestError>>()?; + let timeout = wire + .timeout_seconds + .map(|seconds| { + Duration::try_from_secs_f64(seconds).map_err(|_| OcrRequestError::RequestField { + path: "timeout_seconds".into(), + }) + }) + .transpose()?; + let defaults = OcrConnection::default(); + let request = LiteLLMOcrRequest::new( + wire.model, + document, + wire.custom_llm_provider.as_deref(), + wire.optional_params, + )?; + let connection = OcrConnection { + api_key: nonblank(wire.api_key), + api_base: nonblank(wire.api_base), + extra_headers: headers, + timeout: timeout.unwrap_or(defaults.timeout), + }; + Ok(LiteLLMOcrRequest { + connection, + ..request + }) +} + +fn nonblank(value: Option) -> Option { + value + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} +pub fn decode_request_value( + value: Value, + prefix: &str, +) -> Result { + serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| { + OcrRequestError::RequestField { + path: format!("{prefix}.{}", error.path()), + } + }) +} + +pub fn decode_response( + bytes: &[u8], + native: bool, +) -> Result, OcrResponseError> { + let mut deserializer = serde_json::Deserializer::from_slice(bytes); + let data = serde_path_to_error::deserialize(&mut deserializer).map_err(|error| { + OcrResponseError::ResponseField { + path: error.path().to_string(), + } + })?; + deserializer + .end() + .map_err(|_| OcrResponseError::ResponseField { + path: "response".into(), + })?; + let native = if native { + Some( + serde_json::from_slice(bytes).map_err(|_| OcrResponseError::ResponseField { + path: "response".into(), + })?, + ) + } else { + None + }; + Ok(DecodedOcrResponse { data, native }) +} + +pub fn decode_pre_call_result( + original: OcrPreCallRequest, + value: Value, +) -> Result { + #[derive(Deserialize)] + struct Changed { + document: OcrDocument, + #[serde(default)] + optional_params: Map, + } + let changed: Changed = decode_request_value(value, "guardrail")?; + Ok(OcrPreCallRequest { + document: changed.document, + optional_params: Value::Object(changed.optional_params), + ..original + }) +} + +pub fn decode_during_call_result( + original: OcrDuringCallRequest, + value: Value, +) -> Result { + #[derive(Deserialize)] + struct Changed { + body: Value, + } + let changed: Changed = decode_request_value(value, "guardrail")?; + Ok(OcrDuringCallRequest { + body: changed.body, + ..original + }) +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index d15c032f0bc..2af25a5639a 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; use crate::error::{Error, json_type_name}; use crate::ocr::transformation::{OcrAuthStrategy, OcrProviderConfig, OcrResponseHandling}; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; use serde_json::{Map, Value, json}; use crate::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; @@ -440,7 +440,7 @@ fn transform_document_intelligence_response( model: &str, response_json: Value, preserve_native_response: bool, -) -> Result { +) -> Result { let response = response_json .as_object() .ok_or_else(|| Error::InvalidType { @@ -488,7 +488,7 @@ fn transform_document_intelligence_response( }) .collect(); - Ok(OcrResponseData { + Ok(LiteLLMOcrResponse { usage_info: Some(json!({ "pages_processed": pages.len(), "doc_size_bytes": null, @@ -521,7 +521,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -599,7 +599,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { transform_document_intelligence_response(model, response_json, false) } @@ -608,7 +608,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { model: &str, response_json: Value, optional_params: &Map, - ) -> Result { + ) -> Result { transform_document_intelligence_response( model, response_json, @@ -718,7 +718,7 @@ mod tests { }) } - fn assert_native_fields_preserved(response: &OcrResponseData, operation: &Value) { + fn assert_native_fields_preserved(response: &LiteLLMOcrResponse, operation: &Value) { let analyze_result = &operation["analyzeResult"]; assert_eq!(response.extra_fields["content"], analyze_result["content"]); diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs index 11e8fe7db18..044fc587c22 100644 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs @@ -1,6 +1,6 @@ use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; use serde_json::{Map, Value}; const SUPPORTED_OCR_PARAMS: &[&str] = &[ @@ -107,7 +107,7 @@ impl OcrProviderConfig for MistralOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { let response_object = response_json .as_object() .ok_or_else(|| Error::InvalidType { @@ -128,7 +128,7 @@ impl OcrProviderConfig for MistralOcrConfig { let document_annotation = response_object.get("document_annotation").cloned(); let usage_info = response_object.get("usage_info").cloned(); - Ok(OcrResponseData { + Ok(LiteLLMOcrResponse { pages, model, document_annotation, @@ -178,7 +178,10 @@ pub fn transform_ocr_request( } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub fn transform_ocr_response(model: &str, response_json: Value) -> Result { +pub fn transform_ocr_response( + model: &str, + response_json: Value, +) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs index b8507541e18..f025887846f 100644 --- a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs @@ -6,7 +6,7 @@ use serde_json::{Map, Value, json}; use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; pub const REDUCTO_API_BASE: &str = "https://platform.reducto.ai"; pub const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY"; @@ -283,7 +283,7 @@ fn build_pages(result: &Map) -> Vec { pub fn transform_reducto_response( model: &str, response_json: Value, -) -> Result { +) -> Result { let response = response_json .as_object() .ok_or_else(|| Error::InvalidType { @@ -311,7 +311,7 @@ pub fn transform_reducto_response( "credits": usage.get("credits").cloned().unwrap_or(Value::Null), })); - Ok(OcrResponseData { + Ok(LiteLLMOcrResponse { pages: build_pages(result), model: model.to_string(), document_annotation: None, @@ -341,7 +341,7 @@ impl OcrProviderConfig for ReductoParseV3Config { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { transform_reducto_response(model, response_json) } @@ -383,7 +383,7 @@ impl OcrProviderConfig for ReductoParseLegacyConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { transform_reducto_response(model, response_json) } diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs index c324de8cb45..b0e5a278d0b 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs @@ -1,6 +1,6 @@ use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; +use crate::ocr::types::{LiteLLMOcrResponse, OcrRequestData}; use serde_json::{Map, Value, json}; use crate::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; @@ -226,7 +226,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -301,7 +301,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, model: &str, response_json: Value, - ) -> Result { + ) -> Result { let response = response_json .as_object() .ok_or_else(|| Error::InvalidType { @@ -339,7 +339,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { .get("usage_info") .cloned() .or_else(|| response.get("usage").cloned()); - Ok(OcrResponseData { + Ok(LiteLLMOcrResponse { pages, model: object .get("model") diff --git a/litellm-rust/crates/core/src/url_utils.rs b/litellm-rust/crates/core/src/url_utils.rs new file mode 100644 index 00000000000..982dca0dbe3 --- /dev/null +++ b/litellm-rust/crates/core/src/url_utils.rs @@ -0,0 +1,95 @@ +use std::marker::PhantomData; + +use thiserror::Error; +use url::Url; + +#[derive(Debug, Error)] +pub(crate) enum ApiUrlError { + #[error("invalid URL: {0}")] + Parse(#[from] url::ParseError), + #[error("URL cannot be used as a base")] + CannotBeBase, +} + +pub(crate) struct Base; +pub(crate) struct Complete; + +pub(crate) struct ApiUrl { + url: Url, + state: PhantomData, +} + +impl ApiUrl { + pub(crate) fn parse(value: &str) -> Result { + Ok(Self { + url: Url::parse(value.trim())?, + state: PhantomData, + }) + } + + pub(crate) fn complete_path( + mut self, + target: &[&str], + ) -> Result, ApiUrlError> { + let existing: Vec = self + .url + .path_segments() + .ok_or(ApiUrlError::CannotBeBase)? + .filter(|segment| !segment.is_empty()) + .map(str::to_string) + .collect(); + let overlap = (0..=existing.len().min(target.len())) + .rev() + .find(|&length| { + existing[existing.len() - length..] + .iter() + .map(String::as_str) + .eq(target[..length].iter().copied()) + }) + .unwrap_or(0); + self.url + .path_segments_mut() + .map_err(|()| ApiUrlError::CannotBeBase)? + .pop_if_empty() + .extend(target[overlap..].iter().copied()); + Ok(ApiUrl { + url: self.url, + state: PhantomData, + }) + } +} + +impl ApiUrl { + pub(crate) fn into_string(self) -> String { + self.url.into() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn completion_appends_only_the_missing_path_suffix() { + for (base, expected) in [ + ("https://example.test", "https://example.test/v1/ocr"), + ("https://example.test/v1", "https://example.test/v1/ocr"), + ("https://example.test/v1/ocr", "https://example.test/v1/ocr"), + ] { + let actual = ApiUrl::parse(base) + .and_then(|url| url.complete_path(&["v1", "ocr"])) + .map(|url| url.into_string()) + .expect("url builds"); + assert_eq!(actual, expected); + } + } + + #[test] + fn completion_places_paths_before_queries() { + let actual = ApiUrl::parse("https://example.test/v1?tenant=a") + .and_then(|url| url.complete_path(&["v1", "ocr"])) + .map(|url| url.into_string()) + .expect("url builds"); + assert_eq!(actual, "https://example.test/v1/ocr?tenant=a"); + } +} diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs new file mode 100644 index 00000000000..d1828dfb816 --- /dev/null +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -0,0 +1,227 @@ +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use super::OcrClient; +use super::hooks::{OcrHookFuture, OcrHooks, OcrLogFuture, OcrPreCallRequest}; +use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; +use super::wire::{OcrWireRequest, decode_request}; +use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleTiming}; + +#[test] +fn request_boundary_selects_mistral_and_rejects_unknown_providers() { + let request = OcrWireRequest { + model: "mistral/model".into(), + document: json!({"type":"document_url","document_url":"https://example.com/doc.pdf"}), + api_key: Some("key".into()), + api_base: None, + custom_llm_provider: None, + extra_headers: None, + optional_params: json!({"extract_header":true,"unknown":42}) + .as_object() + .unwrap() + .clone(), + timeout_seconds: None, + }; + assert!(decode_request(request).is_ok()); + assert!( + decode_request(OcrWireRequest { + model: "model".into(), + document: json!({"type":"document_url","document_url":"https://example.com/doc.pdf"}), + api_key: Some("key".into()), + api_base: None, + custom_llm_provider: Some("unknown".into()), + extra_headers: None, + optional_params: serde_json::Map::new(), + timeout_seconds: None, + }) + .is_err() + ); +} + +#[tokio::test] +async fn facade_executes_direct_mistral_once() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "pages":[{"index":0,"markdown":"hello","custom":"preserved"}], + "usage_info":{"pages_processed":1} + }))]) + .await; + let result = perform_ocr(wire_request( + "mistral/model", + &base, + json!({"extract_header":true,"unknown":"ignored"}), + )) + .await + .unwrap(); + server.await.unwrap(); + assert_eq!(result.pages[0]["markdown"], "hello"); + assert_eq!(result.pages[0]["custom"], "preserved"); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("POST /v1/ocr ")); + assert!( + requests[0] + .to_ascii_lowercase() + .contains("authorization: bearer test-key\r\n") + ); + let body: Value = serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!( + body, + json!({ + "model":"model", + "document":{"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}, + "extract_header":true + }) + ); +} + +#[tokio::test] +async fn facade_retains_native_response_when_requested() { + let provider_response = json!({ + "pages":[{"index":0,"markdown":"hello"}], + "usage_info":{"pages_processed":1}, + "provider_only":"preserved" + }); + let (base, _, server) = mock_server(vec![MockResponse::json(provider_response.clone())]).await; + let response = perform_ocr(wire_request( + "mistral/model", + &base, + json!({"req_format":"native"}), + )) + .await + .unwrap(); + + server.await.unwrap(); + assert_eq!(response.provider_native_response, Some(provider_response)); +} + +#[tokio::test] +async fn facade_uses_the_injected_http_client() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let mut default_headers = reqwest::header::HeaderMap::new(); + default_headers.insert( + "x-transport-owner", + reqwest::header::HeaderValue::from_static("host"), + ); + let provider_http = reqwest::Client::builder() + .default_headers(default_headers) + .build() + .unwrap(); + OcrClient::new(provider_http) + .unwrap() + .perform(wire_request("mistral/model", &base, json!({}))) + .await + .unwrap(); + server.await.unwrap(); + assert!(seen.lock().unwrap()[0].contains("x-transport-owner: host")); +} + +struct RecordingHooks { + events: Arc>>, + block: bool, +} + +impl OcrHooks for RecordingHooks { + fn has_guardrails(&self) -> bool { + true + } + + fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> { + Box::pin(async move { + self.events.lock().unwrap().push("pre"); + if self.block { + return Err(crate::Error::InvalidRequest("blocked".into())); + } + Ok(request) + }) + } + + fn during_call( + &self, + request: super::hooks::OcrDuringCallRequest, + ) -> OcrHookFuture<'_, super::hooks::OcrDuringCallRequest> { + Box::pin(async move { + self.events.lock().unwrap().push("during"); + Ok(request) + }) + } + + fn success<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _response: &'a super::LiteLLMOcrResponse, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async move { + self.events.lock().unwrap().push("success"); + }) + } + + fn failure<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _error: &'a crate::Error, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async move { + self.events.lock().unwrap().push("failure"); + }) + } +} + +#[tokio::test] +async fn lifecycle_orders_hooks_and_emits_one_success() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let events = Arc::new(Mutex::new(Vec::new())); + let request = wire_request("mistral/model", &base, json!({})); + let request = super::LiteLLMOcrRequest { + hooks: Arc::new(RecordingHooks { + events: events.clone(), + block: false, + }), + ..request + }; + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(*events.lock().unwrap(), ["pre", "during", "success"]); + assert_eq!(seen.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn lifecycle_blocking_prevents_execution_and_emits_one_failure() { + let events = Arc::new(Mutex::new(Vec::new())); + let request = wire_request("mistral/model", "http://127.0.0.1:1", json!({})); + let request = super::LiteLLMOcrRequest { + hooks: Arc::new(RecordingHooks { + events: events.clone(), + block: true, + }), + ..request + }; + let error = perform_ocr(request).await.unwrap_err(); + assert!(matches!(error, crate::Error::InvalidRequest(_))); + assert_eq!(*events.lock().unwrap(), ["pre", "failure"]); +} + +#[tokio::test] +async fn upstream_failure_emits_one_terminal_failure() { + let (base, seen, server) = mock_server(vec![MockResponse { + status: 500, + headers: vec![], + body: json!({"error":"failed"}), + }]) + .await; + let events = Arc::new(Mutex::new(Vec::new())); + let request = wire_request("mistral/model", &base, json!({})); + let request = super::LiteLLMOcrRequest { + hooks: Arc::new(RecordingHooks { + events: events.clone(), + block: false, + }), + ..request + }; + assert!(perform_ocr(request).await.is_err()); + server.await.unwrap(); + assert_eq!(*events.lock().unwrap(), ["pre", "during", "failure"]); + assert_eq!(seen.lock().unwrap().len(), 1); +} diff --git a/litellm-rust/crates/core/tests/ocr/support.rs b/litellm-rust/crates/core/tests/ocr/support.rs new file mode 100644 index 00000000000..d9720019419 --- /dev/null +++ b/litellm-rust/crates/core/tests/ocr/support.rs @@ -0,0 +1,106 @@ +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +use crate::ocr::wire::{OcrWireRequest, decode_request}; +use crate::ocr::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient}; + +pub(crate) fn ocr_client() -> OcrClient { + OcrClient::for_test(reqwest::Client::new()) +} + +pub(crate) async fn perform_ocr( + request: LiteLLMOcrRequest, +) -> Result { + ocr_client().perform(request).await +} + +pub(crate) fn wire_request(model: &str, base: &str, options: Value) -> LiteLLMOcrRequest { + decode_request(OcrWireRequest { + model: model.into(), + document: json!({"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}), + api_key: Some("test-key".into()), + api_base: Some(base.into()), + custom_llm_provider: None, + extra_headers: None, + optional_params: options.as_object().unwrap().clone(), + timeout_seconds: Some(2.0), + }) + .unwrap() +} + +pub(crate) struct MockResponse { + pub status: u16, + pub headers: Vec<(&'static str, String)>, + pub body: Value, +} + +impl MockResponse { + pub fn json(body: Value) -> Self { + Self { + status: 200, + headers: vec![], + body, + } + } +} + +pub(crate) async fn mock_server( + responses: Vec, +) -> (String, Arc>>, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + let server_base = base.clone(); + let task = tokio::spawn(async move { + for response in responses { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + let mut buffer = [0u8; 4096]; + let header_end = loop { + let n = socket.read(&mut buffer).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buffer[..n]); + if let Some(index) = bytes.windows(4).position(|s| s == b"\r\n\r\n") { + break index + 4; + } + }; + let length = String::from_utf8_lossy(&bytes[..header_end]) + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + while bytes.len() < header_end + length { + let n = socket.read(&mut buffer).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buffer[..n]); + } + seen.lock() + .unwrap() + .push(String::from_utf8_lossy(&bytes).into_owned()); + let body = serde_json::to_vec(&response.body).unwrap(); + let headers = response + .headers + .into_iter() + .map(|(name, value)| { + format!("{name}: {}\r\n", value.replace("{base}", &server_base)) + }) + .collect::(); + let head = format!( + "HTTP/1.1 {} OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n{}\r\n", + response.status, + body.len(), + headers + ); + socket.write_all(head.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); + } + }); + (base, requests, task) +} diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 76c298abf89..407475dedce 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -41,6 +41,7 @@ pub(crate) fn chat_completions_error_to_pyerr(err: Error) -> PyErr { | Error::InvalidRequest(_) | Error::InvalidType { .. } | Error::MissingField(_) + | Error::MissingApiKey { .. } | Error::Routing(_) // Nothing reached the provider, so serving it on Python cannot double // bill and is the only way the caller gets an answer at all. From 93bd3d61b56a167b794c6b8cc2dcc6dcefaf4a47 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:20:49 -0700 Subject: [PATCH 344/411] fix(mcp): preserve verified identity when warming OAuth cache --- .../mcp_server/discoverable_endpoints.py | 1 + .../mcp_server/test_discoverable_endpoints.py | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index c8e02be308b..920fe17e43e 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -634,6 +634,7 @@ async def _store_per_user_token_server_side( server_id=server.server_id, access_token=access_token, ttl=ttl, + identity_binding_proof=identity_binding_proof, ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 42187cb1771..ff649075468 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -11103,3 +11103,46 @@ async def test_identity_bound_authorization_carries_nonce_and_caller_through_cal assert sealed.upstream_code == "upstream-code" assert sealed.oauth_nonce == query["nonce"][0] assert returned["state"] == ["client-state"] + + +@pytest.mark.asyncio +async def test_enforced_login_warms_verified_token_readable_without_database_lookup(monkeypatch): + from types import SimpleNamespace + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import db, mcp_server_manager + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _store_per_user_token_server_side + from litellm.proxy._experimental.mcp_server.oauth2_token_cache import mcp_per_user_token_cache + from litellm.proxy._experimental.mcp_server.oauth_identity_binding import current_binding_proof + from litellm.proxy._experimental.mcp_server.outbound_credentials.dual_cache_token_backend import DualCacheTokenCacheBackend + from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import CachedOAuthTokenStore + from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import LazyPerUserOAuthTokenStore + from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import V2PerUserTokenStore + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv", name="srv", transport="http", auth_type="oauth2", + oauth_identity_binding={"mode": "enforce", "issuer": "https://idp.example", "audiences": ["client"], + "caller_field": "user_id", "principal_claim": "sub"}, + ) + proof = await current_binding_proof(server.oauth_identity_binding, "alice", "srv") + cache = DualCache() + monkeypatch.setenv("LITELLM_SALT_KEY", "test-cache-warm-encryption-salt") + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", SimpleNamespace(invalidate_user_oauth_token_cache=AsyncMock())) + monkeypatch.setattr(db, "store_user_oauth_credential", AsyncMock()) + await _store_per_user_token_server_side( + server=server, user_id="alice", token_response={"access_token": "alice-token", "refresh_token": "private", "expires_in": 3600}, + identity_binding_proof=proof, + ) + read = AsyncMock(return_value=None) + cached = CachedOAuthTokenStore( + V2PerUserTokenStore(read), default_ttl_seconds=300, backend=DualCacheTokenCacheBackend(cache, mcp_per_user_token_cache._codec()), + ) + store = LazyPerUserOAuthTokenStore(lambda _: server, store_builder=lambda _: (cached, True), redis_available=lambda: True) + token = await store.fetch("alice", "srv") + assert token is not None and token.access_token == "alice-token" + assert token.identity_binding_proof == proof + assert token.refresh_token is None + read.assert_not_awaited() From 126dc37de412df372604814b6674fab45635760d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 13:33:36 -0700 Subject: [PATCH 345/411] fix(logging): reconstruct classifier audit redaction payloads --- litellm/integrations/custom_logger.py | 46 ++++------ .../litellm_core_utils/classifier_logging.py | 2 +- litellm/litellm_core_utils/redact_messages.py | 24 ++--- .../spend_tracking/spend_tracking_utils.py | 4 +- .../test_redact_messages.py | 90 ++++++++++++++----- 5 files changed, 96 insertions(+), 70 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 473bf223105..df3237bff07 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -897,10 +897,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac This is useful for logging payloads that contain sensitive information. """ - from copy import copy - import litellm from litellm import Choices, Message, ModelResponse + from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False) excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None) @@ -909,41 +908,25 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac if turn_off_message_logging is False and not excluded_fields: return model_call_details - # Only make a shallow copy of the top-level dict to avoid deepcopy issues - # with complex objects like AuthenticationError that may be present - model_call_details_copy: Final = copy(model_call_details) standard_logging_object: Final = model_call_details.get("standard_logging_object") if standard_logging_object is None: - return model_call_details_copy + return model_call_details.copy() # Make a copy of just the standard_logging_object to avoid modifying the original - standard_logging_object_copy: Final = copy(standard_logging_object) - - # Handle excluded fields - remove them entirely from the payload - if excluded_fields: - for field in excluded_fields: - if field in standard_logging_object_copy: - del standard_logging_object_copy[field] + standard_logging_object_copy: Final = { + key: value + for key, value in standard_logging_object.items() + if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS) + } # Handle turn_off_message_logging - redact messages and responses (if not already excluded) if turn_off_message_logging: - from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit - - for field in CLASSIFIER_AUDIT_FIELDS: - standard_logging_object_copy.pop(field, None) - params: Final = model_call_details_copy.get("litellm_params") - request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None - if isinstance(params, dict) and isinstance(request, dict): - model_call_details_copy["litellm_params"] = { - **params, - "proxy_server_request": without_classifier_audit(request), - } redacted_str: Final = "redacted-by-litellm" - if "messages" not in (excluded_fields or []) and standard_logging_object_copy.get("messages") is not None: + if "messages" not in (excluded_fields or ()) and standard_logging_object_copy.get("messages") is not None: standard_logging_object_copy["messages"] = [Message(content=redacted_str).model_dump()] - if "response" not in (excluded_fields or []) and standard_logging_object_copy.get("response") is not None: + if "response" not in (excluded_fields or ()) and standard_logging_object_copy.get("response") is not None: response: Final = standard_logging_object_copy["response"] # Check if this is a ResponsesAPIResponse (has "output" field) if isinstance(response, dict) and "output" in response: @@ -967,8 +950,15 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac model_response_dict: Final = model_response.model_dump() standard_logging_object_copy["response"] = model_response_dict - model_call_details_copy["standard_logging_object"] = standard_logging_object_copy - return model_call_details_copy + redacted_details: Final = {**model_call_details, "standard_logging_object": standard_logging_object_copy} + params: Final = model_call_details.get("litellm_params") + request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None + if turn_off_message_logging and isinstance(params, dict) and isinstance(request, dict): + return { + **redacted_details, + "litellm_params": {**params, "proxy_server_request": without_classifier_audit(request)}, + } + return redacted_details async def get_proxy_server_request_from_cold_storage_with_object_key( self, diff --git a/litellm/litellm_core_utils/classifier_logging.py b/litellm/litellm_core_utils/classifier_logging.py index c0c93161099..4cbfdb8cd49 100644 --- a/litellm/litellm_core_utils/classifier_logging.py +++ b/litellm/litellm_core_utils/classifier_logging.py @@ -38,7 +38,7 @@ def is_classifier_call(call_type: str, params: Mapping[str, object]) -> bool: def masked_originating_request(request_kwargs: Mapping[str, object] | None) -> Mapping[str, JsonValue] | None: - request: Final = (request_kwargs or {}).get("proxy_server_request") + request: Final = request_kwargs.get("proxy_server_request") if request_kwargs is not None else None body: Final = request.get("body") if isinstance(request, Mapping) else None if not isinstance(body, Mapping): return None diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index c97566c21a8..9d22a5ddef5 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any, Final import litellm from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit +from litellm.litellm_core_utils.classifier_logging import without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) @@ -171,22 +171,13 @@ def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[ payload, but the failure path does not, so a callback that batches both has to redact the ones it is handed. """ - redacted: Final = copy.deepcopy(dict(payload)) # mutable-ok: redacted in place below - _redact_standard_logging_object({"standard_logging_object": redacted}) # mutable-ok: the callee's shape - return redacted + return _redact_standard_logging_object(payload) -def _redact_standard_logging_object(model_call_details: dict): - """Redact messages and response inside standard_logging_object if present.""" - standard_logging_object: Final = model_call_details.get("standard_logging_object") - if standard_logging_object is None: - return - +def _redact_standard_logging_object(payload: Mapping[str, object]) -> dict[str, object]: + standard_logging_object: Final = copy.deepcopy(without_classifier_audit(payload)) redacted_str: Final = REDACTED_BY_LITELLM - for field in CLASSIFIER_AUDIT_FIELDS: - standard_logging_object.pop(field, None) - if standard_logging_object.get("messages") is not None: standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}] @@ -207,6 +198,7 @@ def _redact_standard_logging_object(model_call_details: dict): else: # For other formats (empty dict, None, etc.), use simple text format standard_logging_object["response"] = {"text": redacted_str} + return standard_logging_object def _redact_tool_calls_dict(message: Mapping[str, object]) -> None: @@ -258,8 +250,6 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons copy via redact_streaming_responses_for_custom_logger instead. """ # Redact model_call_details - for field in CLASSIFIER_AUDIT_FIELDS: - model_call_details.pop(field, None) params: Final = model_call_details.get("litellm_params") request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None if isinstance(params, dict) and isinstance(request, Mapping): @@ -267,7 +257,9 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] model_call_details["prompt"] = "" model_call_details["input"] = "" - _redact_standard_logging_object(model_call_details) + standard_logging_object: Final = model_call_details.get("standard_logging_object") + if isinstance(standard_logging_object, Mapping): + model_call_details["standard_logging_object"] = _redact_standard_logging_object(standard_logging_object) redact_vertex_ai_metadata_from_litellm_params(model_call_details) # Redact streaming response diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index da55d1cb153..360b24575bf 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -22,7 +22,7 @@ from litellm.constants import ( from litellm.constants import ( MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, ) -from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields +from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, @@ -1277,7 +1277,7 @@ def _get_proxy_server_request_for_spend_logs_payload( # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): - _request_body = _convert_mapping_to_json_serializable(_request_body) + _request_body = _convert_mapping_to_json_serializable(without_classifier_audit(_request_body)) perform_redaction(model_call_details=_request_body, result=None) _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 1ba9901aab0..76092e96307 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.redact_messages import ( _redact_responses_api_output, perform_redaction, redact_streaming_responses_for_custom_logger, + redacted_standard_logging_payload, should_redact_message_logging, ) from litellm.responses.main import mock_responses_api_response @@ -777,29 +778,6 @@ class TestPerformRedaction: assert response_obj.choices[0].message.content == "secret content" - -@pytest.mark.parametrize("callback_only", [False, True]) -def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None: - audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}} - details: Final = { - "standard_logging_object": {**audit, "messages": [], "response": {}}, - "litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}}, - } - logger: Final = CustomLogger() - logger.turn_off_message_logging = True - if callback_only: - redacted: Final = logger.redact_standard_logging_payload_from_model_call_details(details) - assert "classifier_input" not in redacted["standard_logging_object"] - assert "originating_request_masked" not in redacted["standard_logging_object"] - assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"] - assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"] - assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"] - else: - perform_redaction(details, result=None) - assert "classifier_input" not in details["standard_logging_object"] - assert "originating_request_masked" not in details["standard_logging_object"] - assert "originating_request_masked" not in details["litellm_params"]["proxy_server_request"] - def test_unredactable_result_is_not_deepcopied(self): """A result shape no branch can redact must not be deepcopied. @@ -882,3 +860,69 @@ class TestRedactStreamingResponsesForCustomLogger: assert result_details is model_call_details assert response_obj.choices[0].message.content == "secret content" + + +@pytest.mark.parametrize("callback_only", [False, True]) +def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None: + audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}} + standard_payload: Final = { + **audit, + "messages": [{"role": "user", "content": "private prompt"}], + "response": {"choices": [{"message": {"content": "private answer"}}]}, + "model": "classifier", + } + details: Final = { + "standard_logging_object": standard_payload, + "litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}}, + } + logger: Final = CustomLogger() + logger.turn_off_message_logging = True + if callback_only: + redacted: Final = logger.redact_standard_logging_payload_from_model_call_details(details) + assert "classifier_input" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["standard_logging_object"] + assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"] + assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"] + assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"] + else: + perform_redaction(details, result=None) + assert "classifier_input" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["standard_logging_object"] + assert "originating_request_masked" not in details["litellm_params"]["proxy_server_request"] + + assert standard_payload["classifier_input"] == audit["classifier_input"] + assert standard_payload["originating_request_masked"] == audit["originating_request_masked"] + assert standard_payload["messages"][0]["content"] == "private prompt" + assert standard_payload["response"]["choices"][0]["message"]["content"] == "private answer" + + +@pytest.mark.parametrize("excluded", [False, True]) +def test_classifier_callback_redaction_preserves_exclusions(monkeypatch: pytest.MonkeyPatch, excluded: bool) -> None: + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["messages", "response"] if excluded else []) + payload: Final = { + "classifier_input": {"system": "private rubric"}, + "originating_request_masked": {"input": "private source"}, + "messages": [{"role": "user", "content": "private prompt"}], + "response": {"choices": [{"message": {"content": "private answer"}}]}, + "model": "classifier", + } + logger: Final = CustomLogger() + logger.turn_off_message_logging = True + redacted: Final = logger.redact_standard_logging_payload_from_model_call_details({"standard_logging_object": payload}) + stored: Final = redacted["standard_logging_object"] + assert "classifier_input" not in stored + assert "originating_request_masked" not in stored + assert stored["model"] == "classifier" + assert ("messages" not in stored) is excluded + assert ("response" not in stored) is excluded + if not excluded: + assert stored["messages"][0]["content"] == "redacted-by-litellm" + assert stored["response"]["choices"][0]["message"]["content"] == "redacted-by-litellm" + + failure_payload: Final = redacted_standard_logging_payload(payload) + assert "classifier_input" not in failure_payload + assert "originating_request_masked" not in failure_payload + assert failure_payload["messages"][0]["content"] == "redacted-by-litellm" + assert failure_payload["response"]["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert payload["classifier_input"] == {"system": "private rubric"} + assert payload["response"]["choices"][0]["message"]["content"] == "private answer" From a9cec50960e76f1e311bbbb679d6b3dd3394c93e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:36:23 -0700 Subject: [PATCH 346/411] feat(infra): scale gateway on per-pod RPS and TPS in Helm and Terraform (#40479) * feat(infra): scale gateway on per-pod RPM and TPM in Helm and Terraform Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(helm): require the metrics server before rendering the gateway ServiceMonitor The http port serves /metrics/ behind virtual-key auth, so a ServiceMonitor pointed at it only collects 401s and the RPM/TPM HPA metrics never appear Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(infra): express gateway HPA, KEDA and ECS workload targets per second Rename the per-pod request and token targets in both Helm charts and the AWS module from per minute to per second, and shorten the recommended Prometheus rate window to [1m] with no * 60 so the adapter and KEDA signals are what the HPA compares against. ECS keeps CloudWatch's 60-second aggregation: the ALB target is 60x the per-second variable and the token metric math divides the period Sum by 60 before dividing by the running task count. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- helm/litellm-helm/templates/hpa.yaml | 18 ++ helm/litellm-helm/templates/keda.yaml | 21 ++ helm/litellm-helm/tests/hpa_tests.yaml | 78 +++++++ helm/litellm-helm/tests/keda_tests.yaml | 106 +++++++++ helm/litellm-helm/values.yaml | 36 +++ helm/litellm/templates/gateway/hpa.yaml | 18 ++ .../templates/gateway/servicemonitor.yaml | 28 +++ .../tests/hpa_workload_metrics_tests.yaml | 213 ++++++++++++++++++ helm/litellm/values.yaml | 29 +++ terraform/litellm/aws/README.md | 55 +++++ terraform/litellm/aws/autoscaling.tf | 99 ++++++++ .../aws/tests/workload_autoscaling.tftest.hcl | 194 ++++++++++++++++ terraform/litellm/aws/variables.tf | 41 ++++ terraform/litellm/gcp/README.md | 18 ++ 14 files changed, 954 insertions(+) create mode 100644 helm/litellm-helm/tests/keda_tests.yaml create mode 100644 helm/litellm/templates/gateway/servicemonitor.yaml create mode 100644 helm/litellm/tests/hpa_workload_metrics_tests.yaml create mode 100644 terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl diff --git a/helm/litellm-helm/templates/hpa.yaml b/helm/litellm-helm/templates/hpa.yaml index fec4d1f5c5e..010c4095ab3 100644 --- a/helm/litellm-helm/templates/hpa.yaml +++ b/helm/litellm-helm/templates/hpa.yaml @@ -33,4 +33,22 @@ spec: type: Utilization averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} + {{- with .Values.autoscaling.targetRequestsPerSecond }} + - type: Pods + pods: + metric: + name: litellm_requests_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with .Values.autoscaling.targetTokensPerSecond }} + - type: Pods + pods: + metric: + name: litellm_tokens_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} {{- end }} diff --git a/helm/litellm-helm/templates/keda.yaml b/helm/litellm-helm/templates/keda.yaml index fe5190fffc6..bf585d0d4be 100644 --- a/helm/litellm-helm/templates/keda.yaml +++ b/helm/litellm-helm/templates/keda.yaml @@ -23,6 +23,27 @@ spec: triggers: {{- with .Values.keda.triggers }} {{- toYaml . | nindent 2 }} +{{- end }} +{{- $prom := .Values.keda.prometheus }} +{{- if or $prom.requestsPerSecond $prom.tokensPerSecond }} +{{- if not $prom.serverAddress }} +{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set" }} +{{- end }} +{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s%s" (include "litellm.fullname" .) (ternary "-metrics" "" .Values.metricsServer.enabled)) }} +{{- with $prom.requestsPerSecond }} + - type: prometheus + metadata: + serverAddress: {{ $prom.serverAddress | quote }} + threshold: {{ toJson . | trimAll "\"" | quote }} + query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }} +{{- end }} +{{- with $prom.tokensPerSecond }} + - type: prometheus + metadata: + serverAddress: {{ $prom.serverAddress | quote }} + threshold: {{ toJson . | trimAll "\"" | quote }} + query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }} +{{- end }} {{- end }} advanced: restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }} diff --git a/helm/litellm-helm/tests/hpa_tests.yaml b/helm/litellm-helm/tests/hpa_tests.yaml index cd062dd5971..e446f58c8fe 100644 --- a/helm/litellm-helm/tests/hpa_tests.yaml +++ b/helm/litellm-helm/tests/hpa_tests.yaml @@ -61,6 +61,84 @@ tests: - equal: { path: "spec.metrics[1].resource.name", value: memory } - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 } + - it: "renders no workload metrics by default" + set: + autoscaling.enabled: true + autoscaling.targetMemoryUtilizationPercentage: 80 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - notContains: { path: spec.metrics, content: { type: Pods }, any: true } + + - it: "adds a requests-per-second Pods metric after the cpu metric" + set: + autoscaling.enabled: true + autoscaling.targetRequestsPerSecond: 90 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + - equal: + path: "spec.metrics[1]" + value: + type: Pods + pods: + metric: { name: litellm_requests_per_second } + target: { type: AverageValue, averageValue: "90" } + + - it: "adds a tokens-per-second Pods metric on its own" + set: + autoscaling.enabled: true + autoscaling.targetTokensPerSecond: 6M + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - equal: + path: "spec.metrics[1]" + value: + type: Pods + pods: + metric: { name: litellm_tokens_per_second } + target: { type: AverageValue, averageValue: "6M" } + - notContains: + path: spec.metrics + content: { type: Pods, pods: { metric: { name: litellm_requests_per_second } } } + any: true + + - it: "renders requests, tokens, cpu and memory metrics together" + set: + autoscaling.enabled: true + autoscaling.targetMemoryUtilizationPercentage: 80 + autoscaling.targetRequestsPerSecond: 90 + autoscaling.targetTokensPerSecond: 6000000 + asserts: + - lengthEqual: { path: spec.metrics, count: 4 } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + - equal: { path: "spec.metrics[1].resource.name", value: memory } + - equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_second } + - equal: { path: "spec.metrics[2].pods.target.averageValue", value: "90" } + - equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_second } + - equal: { path: "spec.metrics[3].pods.target.averageValue", value: "6000000" } + + - it: "scales on workload metrics alone when the cpu target is cleared" + set: + autoscaling.enabled: true + autoscaling.targetCPUUtilizationPercentage: null + autoscaling.targetRequestsPerSecond: 90 + autoscaling.targetTokensPerSecond: 6000000 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - notContains: { path: spec.metrics, content: { type: Resource }, any: true } + - equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_second } + - equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_second } + - notMatchRegexRaw: { pattern: per_minute } + + - it: "ignores the per-minute keys, which the chart never shipped" + set: + autoscaling.enabled: true + autoscaling.targetRequestsPerMinute: 5400 + autoscaling.targetTokensPerMinute: 360000000 + asserts: + - lengthEqual: { path: spec.metrics, count: 1 } + - notContains: { path: spec.metrics, content: { type: Pods }, any: true } + - it: "renders no hpa when autoscaling is disabled" asserts: - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/tests/keda_tests.yaml b/helm/litellm-helm/tests/keda_tests.yaml new file mode 100644 index 00000000000..c9598646223 --- /dev/null +++ b/helm/litellm-helm/tests/keda_tests.yaml @@ -0,0 +1,106 @@ +suite: "keda" +templates: + - keda.yaml +release: + name: rel + namespace: llm +tests: + - it: "renders no scaled object by default" + asserts: + - hasDocuments: { count: 0 } + + - it: "passes user triggers through and adds no prometheus triggers by default" + set: + keda.enabled: true + keda.triggers: + - type: cpu + metricType: Utilization + metadata: { value: "60" } + asserts: + - isKind: { of: ScaledObject } + - equal: + path: spec.triggers + value: + - type: cpu + metricType: Utilization + metadata: { value: "60" } + + - it: "scales on release-wide requests per second divided by the per-replica target" + set: + keda.enabled: true + keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090 + keda.prometheus.requestsPerSecond: 90 + asserts: + - lengthEqual: { path: spec.triggers, count: 1 } + - equal: + path: "spec.triggers[0]" + value: + type: prometheus + metadata: + serverAddress: http://prometheus-operated.monitoring.svc:9090 + threshold: "90" + query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m])) + + - it: "scales on tokens per second on its own" + set: + keda.enabled: true + keda.prometheus.serverAddress: http://prom:9090 + keda.prometheus.tokensPerSecond: 6000000 + asserts: + - lengthEqual: { path: spec.triggers, count: 1 } + - equal: { path: "spec.triggers[0].type", value: prometheus } + - equal: { path: "spec.triggers[0].metadata.threshold", value: "6000000" } + - equal: + path: "spec.triggers[0].metadata.query" + value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m])) + + - it: "appends requests and tokens triggers after user triggers and selects the metrics service job" + set: + keda.enabled: true + metricsServer.enabled: true + keda.triggers: + - type: cpu + metricType: Utilization + metadata: { value: "60" } + keda.prometheus.serverAddress: http://prom:9090 + keda.prometheus.requestsPerSecond: 90 + keda.prometheus.tokensPerSecond: 6000000 + asserts: + - lengthEqual: { path: spec.triggers, count: 3 } + - equal: { path: "spec.triggers[0].type", value: cpu } + - equal: { path: "spec.triggers[1].metadata.threshold", value: "90" } + - equal: + path: "spec.triggers[1].metadata.query" + value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) + - equal: { path: "spec.triggers[2].metadata.threshold", value: "6000000" } + - equal: + path: "spec.triggers[2].metadata.query" + value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) + - notMatchRegexRaw: { pattern: "\\* *60|per_minute|PerMinute" } + + - it: "ignores the per-minute keys, which the chart never shipped" + set: + keda.enabled: true + keda.prometheus.serverAddress: http://prom:9090 + keda.prometheus.requestsPerMinute: 5400 + keda.prometheus.tokensPerMinute: 360000000 + asserts: + - isKind: { of: ScaledObject } + - isNullOrEmpty: { path: spec.triggers } + + - it: "refuses a workload target without a prometheus server address" + set: + keda.enabled: true + keda.prometheus.requestsPerSecond: 90 + asserts: + - failedTemplate: + errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set + + - it: "yields to the hpa when both autoscalers are enabled" + set: + autoscaling.enabled: true + keda.enabled: true + keda.prometheus.serverAddress: http://prom:9090 + keda.prometheus.requestsPerSecond: 90 + asserts: + - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 8dc7b967e11..907f85de8fc 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -222,6 +222,25 @@ autoscaling: # Memory is a floor to provision under 'resources', not a signal to scale on. # targetMemoryUtilizationPercentage: 80 # behavior: {} + # Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics + # named `litellm_requests_per_second` and `litellm_tokens_per_second` with an + # AverageValue target, alongside whichever resource targets are set (the HPA + # follows the metric asking for the most replicas). A Prometheus Adapter must + # serve those two names on custom.metrics.k8s.io from the proxy's counters, + # grouped by the scrape target's `pod` label (enable serviceMonitor below so + # every pod is scraped on its own): + # litellm_requests_per_second: + # sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + # litellm_tokens_per_second: + # sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + # rate() over [1m] is already per second, so no `* 60`. How fast the HPA + # reacts is set by that window, the scrape interval and the HPA sync period + # (15s by default), not by the unit: keep serviceMonitor.interval at 15s or + # faster so a 1m window holds at least 4 samples. averageValue takes SI + # suffixes, so "6M" is six million tokens per second per pod. Tokens are + # counted when a response completes, so TPS trails long streams. + targetRequestsPerSecond: "" + targetTokensPerSecond: "" # Autoscaling with keda is mutually exclusive with hpa keda: @@ -243,6 +262,23 @@ keda: # metricName: http_requests_total # threshold: '100' # query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + # First-class Prometheus triggers on the proxy's own request and token + # counters, appended to `triggers`. Each target is the per-second load one + # replica should carry: KEDA divides the release-wide + # `sum(rate([1m]))` by it to pick the replica count. Thresholds + # are plain numbers (KEDA parses them as floats, no SI suffixes). The + # queries select samples by the release namespace and the `job` label the + # chart's ServiceMonitor produces (the metrics Service name), so enable + # serviceMonitor below together with metricsServer: the http port serves + # /metrics/ behind virtual-key auth and answers an unauthenticated scrape + # with 401. Reaction time comes from the [1m] window, the scrape interval + # and pollingInterval above, so keep both at 15s or faster. Tokens are + # counted at completion, so TPS trails long streams. serverAddress is + # required once either target is set. + prometheus: + serverAddress: "" + requestsPerSecond: "" + tokensPerSecond: "" behavior: {} # scaleDown: # stabilizationWindowSeconds: 300 diff --git a/helm/litellm/templates/gateway/hpa.yaml b/helm/litellm/templates/gateway/hpa.yaml index e97cef95ffb..4183079cd31 100644 --- a/helm/litellm/templates/gateway/hpa.yaml +++ b/helm/litellm/templates/gateway/hpa.yaml @@ -30,6 +30,24 @@ spec: type: Utilization averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }} {{- end }} + {{- with .Values.gateway.hpa.targetRequestsPerSecond }} + - type: Pods + pods: + metric: + name: litellm_requests_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with .Values.gateway.hpa.targetTokensPerSecond }} + - type: Pods + pods: + metric: + name: litellm_tokens_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} {{- with .Values.gateway.hpa.behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/helm/litellm/templates/gateway/servicemonitor.yaml b/helm/litellm/templates/gateway/servicemonitor.yaml new file mode 100644 index 00000000000..e1bafa6e388 --- /dev/null +++ b/helm/litellm/templates/gateway/servicemonitor.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.gateway.enabled .Values.gateway.serviceMonitor.enabled }} +{{- if not .Values.gateway.metricsServer.enabled }} +{{- fail "gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled: the http port serves /metrics/ behind virtual-key auth, so an unauthenticated scrape gets 401" }} +{{- end }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "litellm.gateway.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: gateway + {{- with .Values.gateway.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "litellm.gateway.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} + endpoints: + - port: metrics + path: /metrics/ + interval: {{ .Values.gateway.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }} + scheme: http +{{- end }} diff --git a/helm/litellm/tests/hpa_workload_metrics_tests.yaml b/helm/litellm/tests/hpa_workload_metrics_tests.yaml new file mode 100644 index 00000000000..a29c53c5ed9 --- /dev/null +++ b/helm/litellm/tests/hpa_workload_metrics_tests.yaml @@ -0,0 +1,213 @@ +suite: test gateway HPA per-pod requests-per-second and tokens-per-second targets +templates: + - gateway/hpa.yaml + - gateway/servicemonitor.yaml +values: + - ./values/required.yaml +tests: + - it: scales on CPU and memory only by default + template: gateway/hpa.yaml + asserts: + - equal: + path: spec.metrics + value: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + + - it: adds a requests-per-second Pods metric next to the resource metrics + template: gateway/hpa.yaml + set: + gateway.hpa.targetRequestsPerSecond: 90 + asserts: + - lengthEqual: + path: spec.metrics + count: 3 + - contains: + path: spec.metrics + content: + type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - equal: + path: spec.metrics[2] + value: + type: Pods + pods: + metric: + name: litellm_requests_per_second + target: + type: AverageValue + averageValue: "90" + - notContains: + path: spec.metrics + content: + type: Pods + pods: + metric: + name: litellm_tokens_per_second + any: true + + - it: adds a tokens-per-second Pods metric on its own + template: gateway/hpa.yaml + set: + gateway.hpa.targetTokensPerSecond: 6M + asserts: + - lengthEqual: + path: spec.metrics + count: 3 + - equal: + path: spec.metrics[2] + value: + type: Pods + pods: + metric: + name: litellm_tokens_per_second + target: + type: AverageValue + averageValue: "6M" + - notContains: + path: spec.metrics + content: + type: Pods + pods: + metric: + name: litellm_requests_per_second + any: true + + - it: renders requests and tokens targets together and keeps CPU and memory + template: gateway/hpa.yaml + set: + gateway.hpa.targetRequestsPerSecond: 90 + gateway.hpa.targetTokensPerSecond: 6000000 + asserts: + - lengthEqual: + path: spec.metrics + count: 4 + - equal: + path: spec.metrics[0].resource.name + value: cpu + - equal: + path: spec.metrics[1].resource.name + value: memory + - equal: + path: spec.metrics[2].pods.metric.name + value: litellm_requests_per_second + - equal: + path: spec.metrics[3].pods.metric.name + value: litellm_tokens_per_second + - equal: + path: spec.metrics[3].pods.target.averageValue + value: "6000000" + + - it: scales on workload metrics alone when the resource targets are cleared + template: gateway/hpa.yaml + set: + gateway.hpa.targetCPUUtilizationPercentage: null + gateway.hpa.targetMemoryUtilizationPercentage: null + gateway.hpa.targetRequestsPerSecond: 90 + gateway.hpa.targetTokensPerSecond: 6000000 + asserts: + - lengthEqual: + path: spec.metrics + count: 2 + - notContains: + path: spec.metrics + content: + type: Resource + any: true + - equal: + path: spec.metrics[0].pods.metric.name + value: litellm_requests_per_second + - equal: + path: spec.metrics[1].pods.metric.name + value: litellm_tokens_per_second + - notMatchRegexRaw: + pattern: per_minute + + - it: ignores the per-minute keys, which the chart never shipped + template: gateway/hpa.yaml + set: + gateway.hpa.targetRequestsPerMinute: 5400 + gateway.hpa.targetTokensPerMinute: 360000000 + asserts: + - lengthEqual: + path: spec.metrics + count: 2 + - notContains: + path: spec.metrics + content: + type: Pods + any: true + + - it: renders no ServiceMonitor by default + template: gateway/servicemonitor.yaml + asserts: + - hasDocuments: + count: 0 + + - it: refuses a ServiceMonitor without the metrics server, whose http port needs a bearer token + template: gateway/servicemonitor.yaml + set: + gateway.serviceMonitor.enabled: true + asserts: + - failedTemplate: + errorPattern: gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled + + - it: scrapes each gateway pod through the metrics port + template: gateway/servicemonitor.yaml + release: + name: rel + namespace: llm + set: + gateway.serviceMonitor.enabled: true + gateway.metricsServer.enabled: true + gateway.serviceMonitor.labels: + release: kube-prometheus-stack + asserts: + - isKind: + of: ServiceMonitor + - equal: + path: metadata.labels.release + value: kube-prometheus-stack + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: rel + app.kubernetes.io/component: gateway + - equal: + path: spec.namespaceSelector.matchNames + value: + - llm + - equal: + path: spec.endpoints + value: + - port: metrics + path: /metrics/ + interval: 15s + scrapeTimeout: 10s + scheme: http + + - it: honours a custom scrape interval + template: gateway/servicemonitor.yaml + set: + gateway.serviceMonitor.enabled: true + gateway.serviceMonitor.interval: 30s + gateway.metricsServer.enabled: true + asserts: + - equal: + path: spec.endpoints[0].interval + value: 30s diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index b5d535c992d..be1e6d43987 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -284,6 +284,16 @@ gateway: memory: 128Mi limits: memory: 512Mi + # Prometheus Operator ServiceMonitor for the gateway pods. Scrapes the + # `-metrics` Service, so it requires metricsServer above (the http + # port serves /metrics/ behind virtual-key auth). Every pod is its own scrape + # target, so the samples carry the `pod` label the per-pod autoscaling + # queries below group by. + serviceMonitor: + enabled: false + labels: {} + interval: 15s + scrapeTimeout: 10s image: repository: ghcr.io/berriai/litellm-gateway tag: "" # defaults to .Chart.AppVersion @@ -340,6 +350,25 @@ gateway: # policies: # - { type: Percent, value: 100, periodSeconds: 30 } behavior: {} + # Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics + # named `litellm_requests_per_second` and `litellm_tokens_per_second` with an + # AverageValue target. They coexist with the CPU/memory targets above: the + # HPA scales on whichever metric asks for the most replicas. Kubernetes has + # no idea what a token is, so a Prometheus Adapter must serve those two + # names on custom.metrics.k8s.io from the proxy's counters, grouped by the + # scrape target's `pod` label (enable serviceMonitor above): + # litellm_requests_per_second: + # sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + # litellm_tokens_per_second: + # sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) + # rate() over [1m] is already per second, so no `* 60`. How fast the HPA + # reacts is set by that window, the scrape interval and the HPA sync period + # (15s by default), not by the unit: keep serviceMonitor.interval at 15s or + # faster so a 1m window holds at least 4 samples. averageValue takes SI + # suffixes, so "6M" is six million tokens per second per pod. Tokens are + # counted when a response completes, so TPS trails long streams. + targetRequestsPerSecond: "" + targetTokensPerSecond: "" # PodDisruptionBudget for the gateway pods. Set exactly one of # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set; # enabling without either falls back to `maxUnavailable: 1`). Disabled by diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 923a4ca2da8..6d3e0269a15 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -258,6 +258,61 @@ gateway_metrics_port = 4001 gateway_metrics_scrape_cidrs = ["10.0.0.0/16"] ``` +### Scaling the gateway on requests and tokens + +By default the gateway service target-tracks CPU (`gateway_cpu_target`) and +memory (`gateway_memory_target`). Two more targets add workload signals next +to them. Application Auto Scaling evaluates every attached policy and follows +the one asking for the most tasks, so the resource policies keep working as a +floor while requests or tokens drive scale-out + +Both targets are per task per second, the way load is usually quoted (1k +rps, 75M tok/s). CloudWatch is the limit on how fast they react: target +tracking evaluates every metric, predefined or custom, aggregated over +60-second periods and has no period setting, so ECS reacts on a roughly +one-minute cadence whatever unit the variable is written in. The Kubernetes +charts get a faster signal because the Prometheus `rate()` window and scrape +interval are theirs to shorten + +`gateway_target_requests_per_second` adds an `ALBRequestCountPerTarget` +policy on the gateway target group. The ALB publishes that metric as requests +per minute per registered task, so the policy's target value is 60 times the +variable: 90 rps becomes a target of 5,400 per minute. No agent or sidecar is +needed + +`gateway_target_tokens_per_second` adds a metric-math policy over a +CloudWatch metric of the gateway's `litellm_total_tokens_metric_total` +counter and the service's `RunningTaskCount` from Container Insights. Nothing +native to ECS carries token throughput, so you publish that metric yourself +with the CloudWatch agent's Prometheus scraper pointed at the metrics sidecar +above. The agent emits the delta of a counter between scrapes, so `Sum` over +the 60-second period is the tokens served in that minute; the expression +divides by 60 (`tokens_per_second`) and then by the task count +(`tokens_per_second_per_task`). Tokens are counted when a response completes, +so long streams show up late in this signal. `gateway_tokens_metric` tells the +policy where the agent publishes: the namespace, the metric name (defaults to +the counter name) and the dimensions from your `metric_declaration` + +```hcl +gateway_metrics_port = 4001 +gateway_target_requests_per_second = 90 +gateway_target_tokens_per_second = 6000000 +gateway_tokens_metric = { + namespace = "LiteLLM/Prometheus" + dimensions = { ClusterName = "acme-litellm-prod", TaskDefinitionFamily = "acme-litellm-prod-gateway" } +} +``` + +Worked example for the request policy: 1,000 rps across 10 tasks is 100 rps +per task (the ALB reports it as 6,000 per minute per target) against a target +of 90 (5,400), so target tracking sizes the service to +`ceil(10 * 100 / 90) = 12` tasks. The token policy does the same arithmetic: +ten tasks handle 4,200,000,000 tokens in a minute, `tokens / 60` is +70,000,000 tokens per second and `tokens_per_second / running_tasks` is +7,000,000 against a target of 6,000,000, so the service grows to +`ceil(10 * 7000000 / 6000000) = 12`. Container Insights must be enabled on the +cluster for `RunningTaskCount` to exist + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/aws/autoscaling.tf b/terraform/litellm/aws/autoscaling.tf index 71b6c24fac7..5197311d1b3 100644 --- a/terraform/litellm/aws/autoscaling.tf +++ b/terraform/litellm/aws/autoscaling.tf @@ -52,6 +52,105 @@ resource "aws_appautoscaling_policy" "gateway_memory" { } } +resource "aws_appautoscaling_policy" "gateway_requests" { + count = var.gateway_autoscaling_enabled && var.gateway_target_requests_per_second > 0 ? 1 : 0 + name = "${local.name}-gateway-requests" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.gateway[0].service_namespace + resource_id = aws_appautoscaling_target.gateway[0].resource_id + scalable_dimension = aws_appautoscaling_target.gateway[0].scalable_dimension + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "ALBRequestCountPerTarget" + resource_label = "${aws_lb.this.arn_suffix}/${aws_lb_target_group.gateway.arn_suffix}" + } + # ALBRequestCountPerTarget is a per-minute count + target_value = var.gateway_target_requests_per_second * 60 + } +} + +resource "aws_appautoscaling_policy" "gateway_tokens" { + count = var.gateway_autoscaling_enabled && var.gateway_target_tokens_per_second > 0 ? 1 : 0 + name = "${local.name}-gateway-tokens" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.gateway[0].service_namespace + resource_id = aws_appautoscaling_target.gateway[0].resource_id + scalable_dimension = aws_appautoscaling_target.gateway[0].scalable_dimension + + lifecycle { + precondition { + condition = var.gateway_tokens_metric != null + error_message = "gateway_tokens_metric is required when gateway_target_tokens_per_second > 0." + } + } + + target_tracking_scaling_policy_configuration { + target_value = var.gateway_target_tokens_per_second + + # target tracking has no period setting and always aggregates over 60s + customized_metric_specification { + metrics { + id = "tokens" + return_data = false + + metric_stat { + stat = "Sum" + + metric { + namespace = var.gateway_tokens_metric.namespace + metric_name = var.gateway_tokens_metric.name + + dynamic "dimensions" { + for_each = var.gateway_tokens_metric.dimensions + content { + name = dimensions.key + value = dimensions.value + } + } + } + } + } + + metrics { + id = "running_tasks" + return_data = false + + metric_stat { + stat = "Average" + + metric { + namespace = "ECS/ContainerInsights" + metric_name = "RunningTaskCount" + + dimensions { + name = "ClusterName" + value = aws_ecs_cluster.this.name + } + dimensions { + name = "ServiceName" + value = aws_ecs_service.gateway.name + } + } + } + } + + metrics { + id = "tokens_per_second" + expression = "tokens / 60" + return_data = false + } + + metrics { + id = "tokens_per_second_per_task" + expression = "tokens_per_second / running_tasks" + label = "Tokens per second per gateway task" + return_data = true + } + } + } +} + # ---------- Backend ---------- resource "aws_appautoscaling_target" "backend" { count = var.backend_autoscaling_enabled ? 1 : 0 diff --git a/terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl b/terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl new file mode 100644 index 00000000000..94e281faf94 --- /dev/null +++ b/terraform/litellm/aws/tests/workload_autoscaling.tftest.hcl @@ -0,0 +1,194 @@ +# Plan-only coverage for the gateway request and token autoscaling policies. +# Offline via mock_provider, same as byo_infrastructure.tftest.hcl. + +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} +mock_provider "random" {} + +variables { + region = "us-east-1" + tenant = "acme" + env = "test" + allow_plaintext_alb = true + azs = ["us-east-1a", "us-east-1b"] +} + +run "defaults_scale_on_cpu_and_memory_only" { + command = plan + + assert { + condition = alltrue([ + length(aws_appautoscaling_policy.gateway_cpu) == 1, + length(aws_appautoscaling_policy.gateway_memory) == 1, + length(aws_appautoscaling_policy.gateway_requests) == 0, + length(aws_appautoscaling_policy.gateway_tokens) == 0, + ]) + error_message = "Request and token policies must be absent by default while the CPU and memory policies stay." + } +} + +run "requests_per_second_adds_an_alb_request_count_policy" { + command = plan + + variables { + gateway_target_requests_per_second = 90 + } + + assert { + condition = length(aws_appautoscaling_policy.gateway_requests) == 1 && length(aws_appautoscaling_policy.gateway_tokens) == 0 + error_message = "A request target alone must add exactly the request policy." + } + + assert { + condition = alltrue([ + aws_appautoscaling_policy.gateway_requests[0].name == "acme-litellm-test-gateway-requests", + aws_appautoscaling_policy.gateway_requests[0].policy_type == "TargetTrackingScaling", + aws_appautoscaling_policy.gateway_requests[0].service_namespace == "ecs", + aws_appautoscaling_policy.gateway_requests[0].resource_id == "service/acme-litellm-test/acme-litellm-test-gateway", + aws_appautoscaling_policy.gateway_requests[0].scalable_dimension == "ecs:service:DesiredCount", + ]) + error_message = "The request policy must be a target-tracking policy on the gateway service's desired count." + } + + assert { + condition = alltrue([ + one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).target_value == 5400, + one(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).predefined_metric_specification).predefined_metric_type == "ALBRequestCountPerTarget", + length(one(aws_appautoscaling_policy.gateway_requests[0].target_tracking_scaling_policy_configuration).customized_metric_specification) == 0, + ]) + error_message = "The request policy must track ALBRequestCountPerTarget at 60 times the configured requests per second per task." + } +} + +run "tokens_per_second_adds_a_metric_math_policy" { + command = plan + + variables { + gateway_target_tokens_per_second = 6000000 + gateway_tokens_metric = { + namespace = "LiteLLM/Prometheus" + dimensions = { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" } + } + } + + assert { + condition = length(aws_appautoscaling_policy.gateway_tokens) == 1 && length(aws_appautoscaling_policy.gateway_requests) == 0 + error_message = "A token target alone must add exactly the token policy." + } + + assert { + condition = alltrue([ + aws_appautoscaling_policy.gateway_tokens[0].name == "acme-litellm-test-gateway-tokens", + aws_appautoscaling_policy.gateway_tokens[0].resource_id == "service/acme-litellm-test/acme-litellm-test-gateway", + one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).target_value == 6000000, + length(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).predefined_metric_specification) == 0, + ]) + error_message = "The token policy must track a customized metric at the configured tokens per second per task." + } + + assert { + condition = alltrue([ + length(one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics) == 4, + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].id == "tokens", + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].return_data == false, + one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).stat == "Sum", + one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).namespace == "LiteLLM/Prometheus", + one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).metric_name == "litellm_total_tokens_metric_total", + { for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", TaskDefinitionFamily = "acme-litellm-test-gateway" }, + ]) + error_message = "The first metric must sum the published token counter deltas under the configured namespace and dimensions." + } + + assert { + condition = alltrue([ + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].id == "running_tasks", + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].return_data == false, + one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).stat == "Average", + one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).namespace == "ECS/ContainerInsights", + one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).metric_name == "RunningTaskCount", + { for d in one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["running_tasks"].metric_stat).metric).dimensions : d.name => d.value } == { ClusterName = "acme-litellm-test", ServiceName = "acme-litellm-test-gateway" }, + ]) + error_message = "The second metric must read the gateway service's Container Insights RunningTaskCount." + } + + assert { + condition = alltrue([ + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second"].expression == "tokens / 60", + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second"].return_data == false, + ]) + error_message = "The 60s period Sum must be divided by 60 to yield tokens per second." + } + + assert { + condition = alltrue([ + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second_per_task"].expression == "tokens_per_second / running_tasks", + { for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens_per_second_per_task"].return_data == true, + length([for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m if m.return_data]) == 1, + ]) + error_message = "Only the per-task tokens per second may return data to the scaling policy." + } +} + +run "tokens_per_second_needs_the_metric_location" { + command = plan + + variables { + gateway_target_tokens_per_second = 6000000 + } + + expect_failures = [ + aws_appautoscaling_policy.gateway_tokens, + ] +} + +run "requests_and_tokens_scale_next_to_cpu_and_memory" { + command = plan + + variables { + gateway_target_requests_per_second = 90 + gateway_target_tokens_per_second = 6000000 + gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" } + } + + assert { + condition = alltrue([ + length(aws_appautoscaling_policy.gateway_cpu) == 1, + length(aws_appautoscaling_policy.gateway_memory) == 1, + length(aws_appautoscaling_policy.gateway_requests) == 1, + length(aws_appautoscaling_policy.gateway_tokens) == 1, + one(aws_appautoscaling_policy.gateway_cpu[0].target_tracking_scaling_policy_configuration).target_value == 70, + one(aws_appautoscaling_policy.gateway_memory[0].target_tracking_scaling_policy_configuration).target_value == 80, + ]) + error_message = "Workload policies must coexist with the CPU and memory policies at their default targets." + } + + assert { + condition = length(one(one({ for m in one(one(aws_appautoscaling_policy.gateway_tokens[0].target_tracking_scaling_policy_configuration).customized_metric_specification).metrics : m.id => m }["tokens"].metric_stat).metric).dimensions) == 0 + error_message = "Omitting dimensions must query the token metric without any." + } +} + +run "workload_targets_are_ignored_when_autoscaling_is_off" { + command = plan + + variables { + gateway_autoscaling_enabled = false + gateway_target_requests_per_second = 90 + gateway_target_tokens_per_second = 6000000 + gateway_tokens_metric = { namespace = "LiteLLM/Prometheus" } + } + + assert { + condition = alltrue([ + length(aws_appautoscaling_target.gateway) == 0, + length(aws_appautoscaling_policy.gateway_requests) == 0, + length(aws_appautoscaling_policy.gateway_tokens) == 0, + ]) + error_message = "Disabling gateway autoscaling must drop the workload policies with the target." + } +} diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index 667f6db63c9..10a9b392673 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -272,6 +272,47 @@ variable "gateway_memory_target" { default = 80 } +variable "gateway_target_requests_per_second" { + description = <<-EOT + Requests per second one gateway task should serve. Adds an + ALBRequestCountPerTarget target-tracking policy next to the CPU/memory + ones (Application Auto Scaling follows whichever asks for more tasks). + CloudWatch publishes that metric as a 1-minute count, so the policy + targets 60x this value and ECS reacts on a ~1 minute cadence. 0 skips + the policy. + EOT + type = number + default = 0 +} + +variable "gateway_target_tokens_per_second" { + description = <<-EOT + Tokens per second one gateway task should serve. Adds a target-tracking + policy on gateway_tokens_metric summed over each 60s period, divided by + 60 and by the service's Container Insights RunningTaskCount. Tokens are + counted when a response completes, so the signal trails long streams. + 0 skips the policy. + EOT + type = number + default = 0 +} + +variable "gateway_tokens_metric" { + description = <<-EOT + CloudWatch metric carrying the gateway's litellm_total_tokens_metric_total + counter, as published by the CloudWatch agent's Prometheus scraper (it + emits the delta between scrapes, so Sum over a period is the tokens + served in it). Required when gateway_target_tokens_per_second > 0. + dimensions must match the metric_declaration the agent publishes with. + EOT + type = object({ + namespace = string + name = optional(string, "litellm_total_tokens_metric_total") + dimensions = optional(map(string), {}) + }) + default = null +} + variable "backend_autoscaling_enabled" { description = "Toggle Application Auto Scaling target-tracking on the backend service." type = bool diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index c93e5f6b303..00ec7e8afd0 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -238,6 +238,24 @@ this with `litellm_license`. To tune the export cadence, set Behavior matches the AWS stack 1:1; the variable names are identical +### Autoscaling + +Cloud Run scales the gateway on request concurrency (plus its built-in CPU +target), not on a metric you attach. Each instance takes up to +`gateway_max_instance_request_concurrency` requests at once (default 80) +and Cloud Run adds instances between `gateway_min_instances` and +`gateway_max_instances` when the in-flight count fills up. That is the +request-rate signal for this stack: lower the concurrency for LLM streams +that hold a worker for tens of seconds, since a stream counts as one request +for as long as it is open + +There is no tokens-per-second path here. Cloud Run's autoscaler has no +custom-metric input, so the `litellm_total_tokens_metric_total` counter the +proxy exposes cannot drive it. If you need token-based scaling on GCP, run +the gateway on GKE with the Helm chart's `targetTokensPerSecond` (see +"Dependencies only" below) rather than wiring the counter into Cloud +Monitoring, which the autoscaler would ignore + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or From 4de441c18100c39bdcca627b1051af40d9ff2849 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:41:37 -0700 Subject: [PATCH 347/411] test(ui): render TeamSSOSettings under a premium session so the organization dropdown tests fetch --- .../src/components/TeamSSOSettings.test.tsx | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx index 7b3153f6e13..4a042c32cda 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx @@ -8,6 +8,19 @@ import { toast } from "@/lib/toast"; vi.mock("./networking"); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ + token: "test-token", + accessToken: "test-token", + userId: "test-user", + userEmail: "test-user@example.com", + userRole: "Admin", + premiumUser: true, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }), +})); + vi.mock("./common_components/budget_duration_dropdown", () => { const BudgetDurationDropdown = ({ value, onChange }: { value: string | null; onChange: (value: string) => void }) => ( { onChange(event); @@ -232,9 +256,7 @@ const AddPluginForm: React.FC = ({ visible, onClose, accessT "Subfolder path (Optional)", "Path within the repository where the skill lives (e.g., plugins/my-skill). Leave empty if the skill is at the repo root.", )} - description={ - urlEncodesSubdir ? "The URL already points to a subfolder, so this field is disabled" : undefined - } + description={subPathLock ? SUB_PATH_LOCK_REASON[subPathLock] : undefined} > {({ ref, onChange, ...field }) => ( = ({ visible, onClose, accessT onChange(event); recomputePreview(form.getValues("skillUrl"), event.target.value); }} - disabled={urlEncodesSubdir} + disabled={subPathLock !== null} /> )} + {urlPreview?.parsed.source === "archive" && ( + + {({ ref, ...field }) => ( + + )} + + )} + {urlPreview && (
Detected: {urlPreview.label} diff --git a/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx index 2a1530cc352..1104e2c7485 100644 --- a/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx @@ -26,7 +26,7 @@ function getSkillSourceLink(skill: Plugin): { url: string; label: string } | nul const url = src.path ? `${src.url}/tree/main/${src.path}` : src.url; return { url, label: url.replace("https://github.com/", "") }; } - if (src?.source === "url" && src.url) { + if ((src?.source === "url" || src?.source === "archive") && src.url) { return { url: src.url, label: src.url.replace(/^https?:\/\//, "") }; } return null; diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts index 1d07ca09df5..4713fcbc55b 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.test.ts @@ -17,6 +17,7 @@ import { formatKeywords, parseSkillSource, isValidSubPath, + isValidSha256, buildMarketplaceSettingsSnippet, } from "./helpers"; import { MarketplacePluginEntry } from "./types"; @@ -114,6 +115,12 @@ describe("getSourceDisplayText", () => { ); }); + it("shows the archive url for an archive source", () => { + expect(getSourceDisplayText({ source: "archive", url: "https://bucket.s3.amazonaws.com/skill.zip" })).toBe( + "https://bucket.s3.amazonaws.com/skill.zip", + ); + }); + it("returns unknown for missing data", () => { expect(getSourceDisplayText({ source: "github" })).toBe("Unknown source"); }); @@ -140,6 +147,12 @@ describe("getSourceLink", () => { ); }); + it("returns the archive url for an archive source", () => { + expect(getSourceLink({ source: "archive", url: "https://bucket.s3.amazonaws.com/skill.zip" })).toBe( + "https://bucket.s3.amazonaws.com/skill.zip", + ); + }); + it("returns null when no repo or url", () => { expect(getSourceLink({ source: "github" })).toBeNull(); }); @@ -329,6 +342,20 @@ describe("isValidUrl", () => { }); }); +describe("isValidSha256", () => { + it("accepts an empty digest and a 64-character hex digest in either case", () => { + expect(isValidSha256("")).toBe(true); + expect(isValidSha256("a".repeat(64))).toBe(true); + expect(isValidSha256(" " + "ABCDEF0123456789".repeat(4) + " ")).toBe(true); + }); + + it("rejects wrong length and non-hex digests", () => { + expect(isValidSha256("a".repeat(63))).toBe(false); + expect(isValidSha256("a".repeat(65))).toBe(false); + expect(isValidSha256("g".repeat(64))).toBe(false); + }); +}); + describe("parseKeywords", () => { it("splits comma-separated keywords", () => { expect(parseKeywords("a, b, c")).toEqual(["a", "b", "c"]); @@ -445,6 +472,39 @@ describe("parseSkillSource", () => { expect(parseSkillSource("not a url")).toBeNull(); }); + it("parses an S3 zip URL into an archive source and names the skill after the file", () => { + expect(parseSkillSource("https://skills-bucket.s3.us-east-1.amazonaws.com/plugins/My_Skill-1.0.0.zip")).toEqual({ + parsed: { source: "archive", url: "https://skills-bucket.s3.us-east-1.amazonaws.com/plugins/My_Skill-1.0.0.zip" }, + label: "Zip archive — skills-bucket.s3.us-east-1.amazonaws.com/plugins/My_Skill-1.0.0.zip", + suggestedName: "my-skill-1-0-0", + }); + }); + + it("keeps the query string of a zip URL so versioned or signed object links still resolve", () => { + expect(parseSkillSource("https://bucket.s3.amazonaws.com/skill.ZIP?versionId=abc")?.parsed).toEqual({ + source: "archive", + url: "https://bucket.s3.amazonaws.com/skill.ZIP?versionId=abc", + }); + }); + + it("ignores the subfolder for a zip URL since the archive is installed whole", () => { + expect(parseSkillSource("https://artifacts.example.com/skill.zip", "plugins/x")?.parsed).toEqual({ + source: "archive", + url: "https://artifacts.example.com/skill.zip", + }); + }); + + it("rejects a plain http zip URL", () => { + expect(parseSkillSource("http://artifacts.example.com/skill.zip")).toBeNull(); + }); + + it("treats a github zip download URL as an archive rather than a repo path", () => { + expect(parseSkillSource("https://github.com/org/repo/releases/download/v1/skill.zip")?.parsed).toEqual({ + source: "archive", + url: "https://github.com/org/repo/releases/download/v1/skill.zip", + }); + }); + it("suggests a kebab-friendly name from the last path segment", () => { expect(parseSkillSource("github.com/org/my-awesome-skill")?.suggestedName).toBe("my-awesome-skill"); expect(parseSkillSource("github.com/org/repo/tree/main/plugins/cool-skill")?.suggestedName).toBe("cool-skill"); diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts index 5b3d4f20c1e..6f673d67f87 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/helpers.ts @@ -23,6 +23,12 @@ const GITHUB_HOST = "github.com"; const SKILL_FILE_EXTENSION_REGEX = /\.(md|markdown|txt|json|ya?ml|toml)$/i; +const ZIP_ARCHIVE_REGEX = /\.zip$/i; + +export const SHA256_REGEX = /^[0-9a-fA-F]{64}$/; + +export const isValidSha256 = (digest: string): boolean => digest.trim() === "" || SHA256_REGEX.test(digest.trim()); + // WHATWG normalizes obfuscated IPv4 (e.g. 2130706433, 0x7f.0.0.1) to dotted-decimal, so this // catches every IPv4 form; bracketed IPv6 is rejected separately. const IPV4_HOST_REGEX = /^\d{1,3}(\.\d{1,3}){3}$/; @@ -160,16 +166,26 @@ const parseRawGitSource = (url: URL, subPath?: string): SkillSourcePreview | nul }; }; +const parseArchiveSource = (url: URL): SkillSourcePreview => ({ + parsed: { source: "archive", url: url.href }, + label: `Zip archive — ${url.host}${url.pathname}`, + suggestedName: toKebabCase(lastSegment(url.pathname).replace(ZIP_ARCHIVE_REGEX, "")), +}); + /** - * Parse any git-accessible repository URL into a registerable skill source. - * GitHub URLs keep their `github`/`git-subdir` shorthand; every other host is - * treated as a raw repo URL, with an optional subfolder turning it into git-subdir. + * Parse any git-accessible repository URL or https zip archive URL into a registerable skill + * source. A `.zip` path is an `archive` source (S3, Artifactory, any static host). GitHub URLs + * keep their `github`/`git-subdir` shorthand; every other host is treated as a raw repo URL, + * with an optional subfolder turning it into git-subdir. */ export const parseSkillSource = (rawUrl: string, subPath?: string): SkillSourcePreview | null => { const url = parseRepoUrl(rawUrl); if (!url) { return null; } + if (ZIP_ARCHIVE_REGEX.test(url.pathname)) { + return parseArchiveSource(url); + } if (url.hostname.replace(/^www\./, "") === GITHUB_HOST) { return parseGitHubSource(url, subPath); } @@ -245,7 +261,7 @@ export const getSourceDisplayText = (source: PluginSource): string => { if (source.source === "git-subdir" && source.url && source.path) { return `${source.url} @ ${source.path}`; } - if (source.source === "url" && source.url) { + if ((source.source === "url" || source.source === "archive") && source.url) { return source.url; } return "Unknown source"; @@ -258,10 +274,8 @@ export const getSourceLink = (source: PluginSource): string | null => { if (source.source === "github" && source.repo) { return `https://github.com/${source.repo}`; } - if ((source.source === "url" || source.source === "git-subdir") && source.url) { - return source.url; - } - return null; + const linksToUrl = source.source === "url" || source.source === "git-subdir" || source.source === "archive"; + return linksToUrl && source.url ? source.url : null; }; /** diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx index 7cbbb08b623..b37204c6073 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx @@ -26,7 +26,7 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { const src = skill.source; if (src.source === "github" && src.repo) return `https://github.com/${src.repo}`; if (src.source === "git-subdir" && src.url) return src.path ? `${src.url}/tree/main/${src.path}` : src.url; - if (src.source === "url" && src.url) return src.url; + if ((src.source === "url" || src.source === "archive") && src.url) return src.url; return null; })(); diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts index d16c880749b..98f80fd2959 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts @@ -8,10 +8,11 @@ import type { components } from "@/lib/http/schema"; // Kept hand-written: the backend types `source` as Dict[str, str], so the generated type is a // loose string map; this discriminant union is what the parser and display helpers rely on. export interface PluginSource { - source: "github" | "url" | "git-subdir"; + source: "github" | "url" | "git-subdir" | "archive"; repo?: string; // Format: "org/repo" for GitHub url?: string; // Full URL for other sources path?: string; // Subdirectory path for git-subdir + sha256?: string; } export type PluginAuthor = components["schemas"]["PluginAuthor"]; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ab2c1f27a59..c7a760e1eee 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2152,8 +2152,8 @@ export interface paths { * @description Register a new plugin in the LiteLLM marketplace. * * LiteLLM acts as a registry/discovery layer. Plugins are hosted on - * GitHub/GitLab/Bitbucket. Claude Code will clone from the git source - * when users install. + * GitHub/GitLab/Bitbucket or as a zip archive on any https host (e.g. S3). + * Claude Code clones the git source or downloads the archive when users install. * * This endpoint is create-only and never overwrites. If a plugin with * the same name already exists it returns 409 Conflict; use @@ -2163,7 +2163,7 @@ export interface paths { * * Parameters: * - name: Plugin name (kebab-case) - * - source: Git source reference (github, url, or git-subdir format) + * - source: Plugin source reference (github, url, git-subdir, or archive format) * - version: Semantic version (optional) * - description: Plugin description (optional) * - author: Author information (optional) @@ -2229,7 +2229,7 @@ export interface paths { * * Parameters: * - plugin_name: Name of the plugin to update (path parameter) - * - source: Git source reference (github, url, or git-subdir format) + * - source: Plugin source reference (github, url, git-subdir, or archive format) * - version: Semantic version (optional) * - description: Plugin description (optional) * - author: Author information (optional) @@ -33638,7 +33638,7 @@ export interface components { name: string; /** * Source - * @description Git source reference + * @description Plugin source reference */ source: { [key: string]: string; @@ -34887,7 +34887,7 @@ export interface components { * @description Request body for registering a plugin in the marketplace. * * LiteLLM acts as a registry/discovery layer. Plugins are hosted on - * GitHub/GitLab/Bitbucket and referenced by their git source. + * GitHub/GitLab/Bitbucket or as a zip archive on any https host and referenced by their source. */ RegisterPluginRequest: { /** @description Plugin author */ @@ -34929,10 +34929,11 @@ export interface components { namespace?: string | null; /** * Source - * @description Git source reference. Supported formats: + * @description Plugin source reference. Supported formats: * - GitHub: {'source': 'github', 'repo': 'org/repo'} * - Git URL: {'source': 'url', 'url': 'https://github.com/org/repo.git'} * - Git Subdir: {'source': 'git-subdir', 'url': 'https://github.com/org/repo.git', 'path': 'plugins/plugin-name'} + * - Zip archive on any https host (e.g. S3): {'source': 'archive', 'url': 'https://bucket.s3.amazonaws.com/plugin.zip', 'sha256': ''} */ source: { [key: string]: string; @@ -38211,10 +38212,11 @@ export interface components { namespace?: string | null; /** * Source - * @description Git source reference. Supported formats: + * @description Plugin source reference. Supported formats: * - GitHub: {'source': 'github', 'repo': 'org/repo'} * - Git URL: {'source': 'url', 'url': 'https://github.com/org/repo.git'} * - Git Subdir: {'source': 'git-subdir', 'url': 'https://github.com/org/repo.git', 'path': 'plugins/plugin-name'} + * - Zip archive on any https host (e.g. S3): {'source': 'archive', 'url': 'https://bucket.s3.amazonaws.com/plugin.zip', 'sha256': ''} */ source: { [key: string]: string; From 6cc13e07a68f1e822c1acbb7bec808c45baec8e2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:26:09 -0700 Subject: [PATCH 363/411] feat(proxy): granular key/team access control for Claude Code marketplace plugins (#40518) Adds object_permission.skills to keys and teams, enforces it on /claude-code/marketplace.json?key=, /claude-code/plugins and /claude-code/plugins/{name}, and exposes an Allowed Skills selector in the key and team create/edit forms of the Admin UI Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../migration.sql | 1 + .../litellm_proxy_extras/schema.prisma | 1 + litellm/models/object_permission.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 30 ++++- litellm/proxy/_types.py | 4 + .../claude_code_marketplace.py | 43 +++++-- .../claude_code_skill_access.py | 67 +++++++++++ litellm/proxy/schema.prisma | 1 + .../object_permission_repository.py | 6 + litellm/types/object_permission.py | 3 +- schema.prisma | 1 + .../test_claude_code_marketplace.py | 2 +- .../test_claude_code_marketplace.py | 97 +++++++++++++++- .../test_claude_code_skill_access.py | 79 +++++++++++++ .../proxy/auth/test_route_checks.py | 14 +++ .../test_customer_endpoints.py | 1 + .../test_object_permission_utils.py | 23 ++++ .../src/components/Teams.test.tsx | 29 +++++ ui/litellm-dashboard/src/components/Teams.tsx | 46 ++++++++ .../components/object_permissions_view.tsx | 11 ++ .../organisms/createKeyPayload.test.ts | 12 ++ .../components/organisms/createKeyPayload.ts | 5 + .../create_key_button.integration.test.tsx | 24 ++++ .../organisms/create_key_button.tsx | 31 +++++ .../components/skills/SkillSelector.test.tsx | 54 +++++++++ .../src/components/skills/SkillSelector.tsx | 109 ++++++++++++++++++ .../src/components/team/TeamInfo.test.tsx | 43 +++++++ .../src/components/team/TeamInfo.tsx | 26 +++++ .../templates/KeyEditViewControls.tsx | 34 +++++- .../KeyInfoView.handleKeyUpdate.test.tsx | 39 +++++++ .../components/templates/keyEditFormValues.ts | 4 + .../templates/key_edit_view.test.tsx | 40 +++++++ .../components/templates/key_edit_view.tsx | 15 +-- .../components/templates/key_info_view.tsx | 8 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 22 +++- 35 files changed, 892 insertions(+), 34 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_skills_on_object_permission/migration.sql create mode 100644 litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py create mode 100644 tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_skill_access.py create mode 100644 ui/litellm-dashboard/src/components/skills/SkillSelector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/skills/SkillSelector.tsx diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_skills_on_object_permission/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_skills_on_object_permission/migration.sql new file mode 100644 index 00000000000..c982bc38a69 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_skills_on_object_permission/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "skills" TEXT[] DEFAULT ARRAY[]::TEXT[]; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 05c5aad9303..817df082d8c 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -282,6 +282,7 @@ model LiteLLM_ObjectPermissionTable { mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user search_tools String[] @default([]) // search_tool_name values this key/team/user may call mcp_tool_search_enabled Boolean? + skills String[] @default([]) // Claude Code plugin names granted to this key/team beyond the public (enabled) set teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/litellm/models/object_permission.py b/litellm/models/object_permission.py index a09d50ddc33..f178c5ad47f 100644 --- a/litellm/models/object_permission.py +++ b/litellm/models/object_permission.py @@ -23,3 +23,4 @@ class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase): blocked_tools: list[str] | None = [] search_tools: list[str] | None = [] mcp_tool_search_enabled: bool | None = None + skills: list[str] | None = None diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 3f060f1f5f6..f0af17ab818 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -5721,8 +5721,26 @@ "paths": { "/claude-code/marketplace.json": { "get": { - "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin install my-plugin@litellm\n ```", + "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add \n- claude plugin install @\n\nWithout `key` the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` on the key or its team.\n\nReturns:\n Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n ```bash\n claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n claude plugin install my-plugin@litellm\n ```", "operationId": "get_marketplace_claude_code_marketplace_json_get", + "parameters": [ + { + "in": "query", + "name": "key", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key" + } + } + ], "responses": { "200": { "content": { @@ -5731,6 +5749,16 @@ } }, "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" } }, "summary": "Get Marketplace", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f496140e905..23dc8237160 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -497,6 +497,9 @@ class LiteLLMRoutes(enum.Enum): "/v1/messages/count_tokens", "/v1/skills", "/v1/skills/{skill_id}", + "/claude-code/marketplace.json", + "/claude-code/plugins", + "/claude-code/plugins/{plugin_name}", ] # MCP tool-call / passthrough routes — data-plane. Gated by DISABLE_LLM_API_ENDPOINTS. @@ -1124,6 +1127,7 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): models: list[str] | None = None search_tools: list[str] | None = None mcp_tool_search_enabled: bool | None = None + skills: list[str] | None = None from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E402 diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index cdeb1c6a111..7c6a4571948 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -7,10 +7,10 @@ Actual plugin files are hosted on GitHub/GitLab/Bitbucket or as a zip archive on any HTTPS host (S3, Artifactory, a static file server). Endpoints: -/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (unauthenticated) +/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (unauthenticated; `?key=` adds the key's granted skills) /claude-code/plugins - POST - Register a new plugin (create-only, proxy admin only) -/claude-code/plugins - GET - List plugins (any authenticated key) -/claude-code/plugins/{name} - GET - Get plugin details (any authenticated key) +/claude-code/plugins - GET - List plugins visible to the key (enabled, plus granted disabled ones) +/claude-code/plugins/{name} - GET - Get plugin details (403 on a disabled plugin the key is not granted) /claude-code/plugins/{name} - PUT - Update an existing plugin (proxy admin only) /claude-code/plugins/{name}/enable - POST - Enable a plugin (proxy admin only) /claude-code/plugins/{name}/disable - POST - Disable a plugin (proxy admin only) @@ -24,11 +24,15 @@ from datetime import datetime, timezone from typing import Annotated, Final, Protocol, TypedDict from urllib.parse import urlsplit -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import JSONResponse from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, ProxyException, UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( + SkillVisibility, + skill_visibility, +) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.repositories.table_repositories import ClaudeCodePluginRepository @@ -84,7 +88,7 @@ async def _get_prisma_client() -> object: "/claude-code/marketplace.json", tags=["Claude Code Marketplace"], ) -async def get_marketplace(): +async def get_marketplace(request: Request, key: str | None = None): """ Serve marketplace.json for Claude Code plugin discovery. @@ -92,24 +96,35 @@ async def get_marketplace(): - claude plugin marketplace add - claude plugin install @ + Without `key` the catalog holds the enabled (public) plugins. With `?key=sk-...` + the key is authenticated and the catalog also holds the disabled plugins granted + to it through `object_permission.skills` on the key or its team. + Returns: Marketplace catalog with list of available plugins and their git sources. Example: ```bash claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json + claude plugin marketplace add "http://localhost:4000/claude-code/marketplace.json?key=sk-..." claude plugin install my-plugin@litellm ``` """ try: prisma_client: Final = await _get_prisma_client() + caller: Final[UserAPIKeyAuth | None] = ( + await user_api_key_auth(request=request, api_key=f"Bearer {key}") if key else None + ) + visibility: Final[SkillVisibility] = skill_visibility(caller) plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( - where={"enabled": True} + where=visibility.where() ) plugin_list: Final = [] for plugin in plugins: + if not visibility.allows(plugin): + continue try: manifest: Mapping[str, object] = json.loads(plugin.manifest_json or "{}") except json.JSONDecodeError: @@ -149,7 +164,7 @@ async def get_marketplace(): return JSONResponse(content=marketplace) - except HTTPException: + except (HTTPException, ProxyException): raise except Exception as e: verbose_proxy_logger.exception("Error generating marketplace: %s", e) @@ -395,13 +410,15 @@ async def list_plugins( try: prisma_client: Final = await _get_prisma_client() - where: Final = {"enabled": True} if enabled_only else {} + visibility: Final[SkillVisibility] = skill_visibility(user_api_key_dict) plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( - where=where + where={"enabled": True} if enabled_only else visibility.where() ) plugin_list: Final = [] for p in plugins: + if not visibility.allows(p): + continue # Parse manifest to get additional fields manifest = json.loads(p.manifest_json) if p.manifest_json else {} @@ -473,6 +490,12 @@ async def get_plugin( detail={"error": f"Plugin '{plugin_name}' not found"}, ) + if not skill_visibility(user_api_key_dict).allows(plugin): + raise HTTPException( + status_code=403, + detail={"error": f"Plugin '{plugin_name}' is not granted to this key"}, + ) + manifest: Final[Mapping[str, object]] = json.loads(plugin.manifest_json or "{}") if plugin.manifest_json else {} return { diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py new file mode 100644 index 00000000000..e1e1ee6f160 --- /dev/null +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py @@ -0,0 +1,67 @@ +""" +Claude Code marketplace visibility: enabled plugins are public, disabled plugins +are private and resolve only for proxy admins or keys granted them via +``object_permission.skills``. +""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final, Protocol + +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth +from litellm.proxy.common_utils.resource_ownership import is_proxy_admin + +if TYPE_CHECKING: + from prisma.types import LiteLLM_ClaudeCodePluginTableWhereInput + + +class _SkillRecord(Protocol): + name: str + enabled: bool + + +def _skills_of(permission: LiteLLM_ObjectPermissionTable | None) -> frozenset[str]: + return frozenset(permission.skills or ()) if permission is not None else frozenset() + + +def granted_skills(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]: + """Key grant intersected with the team grant when both are non-empty; either alone applies as is. + + An empty list is the Prisma column default for every object-permission row, so it means + "no private grants configured here" and defers to the other scope, same as the agents check. + """ + key_skills: Final = _skills_of(user_api_key_dict.object_permission) + team_skills: Final = _skills_of(user_api_key_dict.team_object_permission) + match (bool(key_skills), bool(team_skills)): + case (True, True): + return key_skills & team_skills + case (True, False): + return key_skills + case _: + return team_skills + + +@dataclass(frozen=True, slots=True) +class SkillVisibility: + granted: frozenset[str] + sees_private: bool + + def allows(self, skill: _SkillRecord) -> bool: + return skill.enabled or self.sees_private or skill.name in self.granted + + def where(self) -> "LiteLLM_ClaudeCodePluginTableWhereInput": + if self.sees_private: + return {} + if not self.granted: + return {"enabled": True} + return {"OR": [{"enabled": True}, {"name": {"in": sorted(self.granted)}}]} + + +PUBLIC_ONLY: Final = SkillVisibility(granted=frozenset(), sees_private=False) + + +def skill_visibility(user_api_key_dict: UserAPIKeyAuth | None) -> SkillVisibility: + if user_api_key_dict is None: + return PUBLIC_ONLY + if is_proxy_admin(user_api_key_dict): + return SkillVisibility(granted=frozenset(), sees_private=True) + return SkillVisibility(granted=granted_skills(user_api_key_dict), sees_private=False) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 05c5aad9303..817df082d8c 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -282,6 +282,7 @@ model LiteLLM_ObjectPermissionTable { mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user search_tools String[] @default([]) // search_tool_name values this key/team/user may call mcp_tool_search_enabled Boolean? + skills String[] @default([]) // Claude Code plugin names granted to this key/team beyond the public (enabled) set teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/litellm/repositories/object_permission_repository.py b/litellm/repositories/object_permission_repository.py index 6b1f9c68e47..7736939c696 100644 --- a/litellm/repositories/object_permission_repository.py +++ b/litellm/repositories/object_permission_repository.py @@ -40,6 +40,7 @@ class ObjectPermissionRepository(BaseRepository[LiteLLM_ObjectPermissionTable]): blocked_tools: list[str] | None = None, mcp_toolsets: list[str] | None = None, search_tools: list[str] | None = None, + skills: list[str] | None = None, ) -> LiteLLM_ObjectPermissionTable: """Create a new object permission record.""" data: Final[dict[str, Any]] = {} @@ -63,6 +64,8 @@ class ObjectPermissionRepository(BaseRepository[LiteLLM_ObjectPermissionTable]): data["mcp_toolsets"] = mcp_toolsets if search_tools is not None: data["search_tools"] = search_tools + if skills is not None: + data["skills"] = skills return await self.create(data) @@ -79,6 +82,7 @@ class ObjectPermissionRepository(BaseRepository[LiteLLM_ObjectPermissionTable]): blocked_tools: list[str] | None = None, mcp_toolsets: list[str] | None = None, search_tools: list[str] | None = None, + skills: list[str] | None = None, ) -> LiteLLM_ObjectPermissionTable | None: """Update an object permission record.""" data: Final[dict[str, Any]] = {} @@ -102,6 +106,8 @@ class ObjectPermissionRepository(BaseRepository[LiteLLM_ObjectPermissionTable]): data["mcp_toolsets"] = mcp_toolsets if search_tools is not None: data["search_tools"] = search_tools + if skills is not None: + data["skills"] = skills return await self.update(object_permission_id, data, id_field="object_permission_id") diff --git a/litellm/types/object_permission.py b/litellm/types/object_permission.py index 1b391a3a1ef..68661aed891 100644 --- a/litellm/types/object_permission.py +++ b/litellm/types/object_permission.py @@ -8,7 +8,7 @@ can adopt the type without violating the SDK-must-not-import-from-proxy layering rule. """ -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class ObjectPermissionDict(TypedDict, total=False): @@ -23,3 +23,4 @@ class ObjectPermissionDict(TypedDict, total=False): models: list[str] | None search_tools: list[str] | None mcp_tool_search_enabled: bool | None + skills: ReadOnly[list[str] | None] diff --git a/schema.prisma b/schema.prisma index 05c5aad9303..817df082d8c 100644 --- a/schema.prisma +++ b/schema.prisma @@ -282,6 +282,7 @@ model LiteLLM_ObjectPermissionTable { mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user search_tools String[] @default([]) // search_tool_name values this key/team/user may call mcp_tool_search_enabled Boolean? + skills String[] @default([]) // Claude Code plugin names granted to this key/team beyond the public (enabled) set teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] verification_tokens LiteLLM_VerificationToken[] diff --git a/tests/pass_through_unit_tests/test_claude_code_marketplace.py b/tests/pass_through_unit_tests/test_claude_code_marketplace.py index 2ca81f1d5d3..bedb8830559 100644 --- a/tests/pass_through_unit_tests/test_claude_code_marketplace.py +++ b/tests/pass_through_unit_tests/test_claude_code_marketplace.py @@ -216,7 +216,7 @@ async def test_get_marketplace(mock_prisma_client): ) # Now get the marketplace - response = await get_marketplace() + response = await get_marketplace(request=MagicMock()) # Response is a JSONResponse, get the body body = json.loads(response.body.decode()) diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py index aa2da262bae..a7c2bd7ba20 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py @@ -11,17 +11,20 @@ from fastapi import HTTPException from unittest.mock import AsyncMock, MagicMock import litellm -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, ProxyException, UserAPIKeyAuth from litellm.proxy.proxy_server import LitellmUserRoles from litellm.types.proxy.claude_code_endpoints import ( RegisterPluginRequest, UpdatePluginRequest, ) +from litellm.proxy.anthropic_endpoints.claude_code_endpoints import claude_code_marketplace from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( delete_plugin, disable_plugin, enable_plugin, get_marketplace, + get_plugin, + list_plugins, register_plugin, update_plugin, ) @@ -38,11 +41,15 @@ def _make_mock_prisma(): async def _find_unique(where): return store.get(where.get("name")) + def _matches(record, where) -> bool: + if "OR" in where: + return any(_matches(record, clause) for clause in where["OR"]) + if "enabled" in where and record.enabled != where["enabled"]: + return False + return "name" not in where or record.name in where["name"]["in"] + async def _find_many(where=None): - records = list(store.values()) - if where and "enabled" in where: - return [r for r in records if r.enabled == where["enabled"]] - return records + return [r for r in store.values() if _matches(r, where or {})] async def _create(data): record = MagicMock() @@ -52,6 +59,8 @@ def _make_mock_prisma(): record.description = data.get("description") record.manifest_json = data.get("manifest_json", "{}") record.enabled = data.get("enabled", True) + record.created_at = data.get("created_at") + record.updated_at = data.get("updated_at") store[data["name"]] = record return record @@ -271,13 +280,89 @@ async def test_get_marketplace_skips_plugin_with_null_manifest(): table = litellm.proxy.proxy_server.prisma_client.db.litellm_claudecodeplugintable await table.create(data={"name": "null-manifest-plugin", "manifest_json": None, "enabled": True}) - response = await get_marketplace() + response = await get_marketplace(request=MagicMock()) assert response.status_code == 200 body = json.loads(response.body) assert [plugin["name"] for plugin in body["plugins"]] == ["good-plugin"] +def _granted_user(skills: list[str]) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + api_key="sk-granted", + user_id="granted-user", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="perm-1", skills=skills), + ) + + +async def _register_public_and_private_plugins() -> None: + for name, enabled in (("public-skill", True), ("private-skill", False)): + await register_plugin( + request=RegisterPluginRequest(name=name, source=_GIT_SUBDIR_SOURCE, version="1.0.0"), + user_api_key_dict=_USER, + ) + if not enabled: + await disable_plugin(plugin_name=name, user_api_key_dict=_USER) + + +async def _listed_names(user: UserAPIKeyAuth) -> set[str]: + response = await list_plugins(user_api_key_dict=user) + return {plugin.name for plugin in response.plugins} + + +@pytest.mark.asyncio +async def test_list_plugins_shows_disabled_plugin_only_to_granted_key_or_admin(): + await _register_public_and_private_plugins() + + assert await _listed_names(_NON_ADMIN_USER) == {"public-skill"} + assert await _listed_names(_granted_user(["other-skill"])) == {"public-skill"} + assert await _listed_names(_granted_user(["private-skill"])) == {"public-skill", "private-skill"} + assert await _listed_names(_USER) == {"public-skill", "private-skill"} + + +@pytest.mark.asyncio +async def test_get_plugin_returns_403_for_disabled_plugin_the_key_is_not_granted(): + await _register_public_and_private_plugins() + + with pytest.raises(HTTPException) as exc_info: + await get_plugin(plugin_name="private-skill", user_api_key_dict=_NON_ADMIN_USER) + assert exc_info.value.status_code == 403 + + assert (await get_plugin(plugin_name="public-skill", user_api_key_dict=_NON_ADMIN_USER))["name"] == "public-skill" + granted = await get_plugin(plugin_name="private-skill", user_api_key_dict=_granted_user(["private-skill"])) + assert granted["name"] == "private-skill" + assert granted["enabled"] is False + + +async def _marketplace_names(key: str | None) -> list[str]: + response = await get_marketplace(request=MagicMock(), key=key) + assert response.status_code == 200 + return sorted(plugin["name"] for plugin in json.loads(response.body)["plugins"]) + + +@pytest.mark.asyncio +async def test_get_marketplace_key_query_param_adds_granted_disabled_plugins(monkeypatch): + await _register_public_and_private_plugins() + keys = {"sk-granted": _granted_user(["private-skill"]), "sk-plain": _NON_ADMIN_USER} + + async def _fake_auth(request, api_key: str) -> UserAPIKeyAuth: + token = api_key.removeprefix("Bearer ") + if token not in keys: + raise ProxyException(message="invalid key", type="auth_error", param="key", code=401) + return keys[token] + + monkeypatch.setattr(claude_code_marketplace, "user_api_key_auth", _fake_auth) + + assert await _marketplace_names(None) == ["public-skill"] + assert await _marketplace_names("sk-plain") == ["public-skill"] + assert await _marketplace_names("sk-granted") == ["private-skill", "public-skill"] + + with pytest.raises(ProxyException) as exc_info: + await get_marketplace(request=MagicMock(), key="sk-bogus") + assert exc_info.value.code == "401" + + @pytest.mark.asyncio async def test_register_plugin_git_subdir_missing_url(): """git-subdir without url field raises HTTP 400.""" diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_skill_access.py b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_skill_access.py new file mode 100644 index 00000000000..dc4016d6db0 --- /dev/null +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_skill_access.py @@ -0,0 +1,79 @@ +"""Unit tests for claude_code_skill_access.py: key/team grant resolution.""" + +from unittest.mock import MagicMock + +import pytest + +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( + granted_skills, + skill_visibility, +) + + +def _perm(skills: list[str] | None) -> LiteLLM_ObjectPermissionTable: + return LiteLLM_ObjectPermissionTable(object_permission_id="perm", skills=skills) + + +def _key(key_skills: list[str] | None, team_skills: list[str] | None) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="sk-user", + user_role=LitellmUserRoles.INTERNAL_USER, + object_permission=_perm(key_skills) if key_skills is not None else None, + team_object_permission=_perm(team_skills) if team_skills is not None else None, + ) + + +def _plugin(name: str, enabled: bool) -> MagicMock: + record = MagicMock() + record.name = name + record.enabled = enabled + return record + + +@pytest.mark.parametrize( + ("key_skills", "team_skills", "expected"), + [ + (None, None, frozenset()), + ([], [], frozenset()), + (["a", "b"], None, frozenset({"a", "b"})), + (None, ["a", "b"], frozenset({"a", "b"})), + (["a", "b"], ["b", "c"], frozenset({"b"})), + (["a"], ["c"], frozenset()), + (["a", "b"], [], frozenset({"a", "b"})), + ([], ["a", "b"], frozenset({"a", "b"})), + ], +) +def test_granted_skills_intersects_key_with_team(key_skills, team_skills, expected): + assert granted_skills(_key(key_skills, team_skills)) == expected + + +def test_visibility_enabled_plugin_is_public_for_everyone(): + public = _plugin("public-skill", enabled=True) + + assert skill_visibility(None).allows(public) + assert skill_visibility(_key(None, None)).allows(public) + assert skill_visibility(_key([], ["other"])).allows(public) + + +def test_visibility_disabled_plugin_needs_grant_or_admin(): + private = _plugin("private-skill", enabled=False) + admin = UserAPIKeyAuth(api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN) + + assert not skill_visibility(None).allows(private) + assert not skill_visibility(_key(None, None)).allows(private) + assert not skill_visibility(_key(["other-skill"], None)).allows(private) + assert not skill_visibility(_key(["private-skill"], ["other-skill"])).allows(private) + assert skill_visibility(_key(["private-skill"], None)).allows(private) + assert skill_visibility(_key(None, ["private-skill"])).allows(private) + assert skill_visibility(admin).allows(private) + + +def test_where_clause_bounds_the_plugin_query_to_what_the_caller_may_see(): + admin = UserAPIKeyAuth(api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN) + + assert skill_visibility(None).where() == {"enabled": True} + assert skill_visibility(_key(None, None)).where() == {"enabled": True} + assert skill_visibility(_key(["b", "a"], None)).where() == {"OR": [{"enabled": True}, {"name": {"in": ["a", "b"]}}]} + assert skill_visibility(_key(["a", "b"], ["b"])).where() == {"OR": [{"enabled": True}, {"name": {"in": ["b"]}}]} + assert skill_visibility(admin).where() == {} diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 0fd19f518d9..4f2ff1582dd 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -3826,3 +3826,17 @@ def test_team_disable_logging_stays_proxy_admin_only(): def test_neighbouring_team_routes_stay_closed(route): """The grant is the callback paths and nothing else on the team namespace.""" assert "Only proxy admin" in _gate(route, LitellmUserRoles.INTERNAL_USER.value) + + +@pytest.mark.parametrize( + "route", + [ + "/claude-code/marketplace.json", + "/claude-code/plugins", + "/claude-code/plugins/my-skill", + ], +) +def test_claude_code_marketplace_routes_open_to_internal_users(route): + """Per-skill visibility is enforced inside the handler, so the route gate must let non-admins through.""" + assert RouteChecks.is_llm_api_route(route) is True + assert _gate(route, LitellmUserRoles.INTERNAL_USER.value) == "allowed" diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py index 1225cb80224..bd59a82cbd2 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py @@ -763,6 +763,7 @@ _EXPECTED_CUSTOMER = { "blocked_tools": [], "search_tools": [], "mcp_tool_search_enabled": None, + "skills": None, }, } diff --git a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py index d7ebb1f60bf..078315c2bf8 100644 --- a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py +++ b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py @@ -122,6 +122,29 @@ async def test_set_object_permission_persists_mcp_tool_search_enabled(): assert created_data["mcp_tool_search_enabled"] is True +@pytest.mark.asyncio +async def test_set_object_permission_persists_skills(): + mock_prisma_client = MagicMock() + mock_created_permission = MagicMock() + mock_created_permission.object_permission_id = "perm_id" + mock_prisma_client.db.litellm_objectpermissiontable.create = AsyncMock( + return_value=mock_created_permission + ) + + data_json = { + "object_permission": LiteLLM_ObjectPermissionBase(skills=["private-skill"]).model_dump(), + } + + await _set_object_permission(data_json=data_json, prisma_client=mock_prisma_client) + + created_data = ( + mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs[ + "data" + ] + ) + assert created_data["skills"] == ["private-skill"] + + # ---- Tests for _extract_requested_mcp_server_ids ---- diff --git a/ui/litellm-dashboard/src/components/Teams.test.tsx b/ui/litellm-dashboard/src/components/Teams.test.tsx index c7df35197e6..70c30596c75 100644 --- a/ui/litellm-dashboard/src/components/Teams.test.tsx +++ b/ui/litellm-dashboard/src/components/Teams.test.tsx @@ -33,6 +33,14 @@ vi.mock("./mcp_server_management/MCPServerSelector", () => ({ ), })); +vi.mock("./skills/SkillSelector", () => ({ + default: ({ onChange }: { onChange: (selected: string[]) => void }) => ( + + ), +})); + const can = vi.fn(); vi.mock("@/app/(dashboard)/hooks/useCan", () => ({ default: (...args: unknown[]) => can(...args), @@ -1359,6 +1367,27 @@ describe("Teams - the exact bytes the create call sends", () => { }); }); + it("puts the selected skills into object_permission.skills and drops the form key", async () => { + await openCreateModal(); + await openSection("Skill Settings", /Allowed Skills/); + fireEvent.click(screen.getByTestId("select-private-skill")); + + const payload = await submit(); + + expect(payload.object_permission).toStrictEqual({ skills: ["private-skill"] }); + expect(payload).not.toHaveProperty("object_permission_skills"); + }); + + it("sends no object_permission when Skill Settings is opened but nothing is selected", async () => { + await openCreateModal(); + await openSection("Skill Settings", /Allowed Skills/); + + const payload = await submit(); + + expect(payload).not.toHaveProperty("object_permission"); + expect(payload).not.toHaveProperty("object_permission_skills"); + }); + it("includes selected MCP toolsets in the create object permission", async () => { await openCreateModal(); await openSection("MCP Settings", /Allowed MCP Servers/); diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index c4060163c78..d2d18139893 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -50,6 +50,7 @@ import { Organization, getDefaultTeamSettings, getGuardrailsList, getPoliciesLis import NumericalInput from "./shared/numerical_input"; import VectorStoreSelector from "./vector_store_management/VectorStoreSelector"; import SearchToolSelector from "./search_tools/SearchToolSelector"; +import SkillSelector from "./skills/SkillSelector"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; interface TeamProps { @@ -99,6 +100,7 @@ const teamCreateFieldsSchema = z.object({ mcp_tool_permissions: z.record(z.string(), z.array(z.string())).optional(), allowed_agents_and_groups: z.object({ agents: z.array(z.string()), accessGroups: z.array(z.string()) }).optional(), object_permission_search_tools: z.array(z.string()).optional(), + object_permission_skills: z.array(z.string()).optional(), }); type TeamCreateFormValues = z.infer; @@ -128,6 +130,7 @@ const EMPTY_TEAM_CREATE_VALUES: TeamCreateFormValues = { mcp_tool_permissions: {}, allowed_agents_and_groups: undefined, object_permission_search_tools: undefined, + object_permission_skills: undefined, }; const ADDITIONAL_SETTINGS_FIELDS = [ @@ -147,6 +150,7 @@ const ADDITIONAL_SETTINGS_FIELDS = [ const MCP_SETTINGS_FIELDS = ["allowed_mcp_servers_and_groups", "mcp_tool_permissions"] as const; const AGENT_SETTINGS_FIELDS = ["allowed_agents_and_groups"] as const; const SEARCH_TOOL_SETTINGS_FIELDS = ["object_permission_search_tools"] as const; +const SKILL_SETTINGS_FIELDS = ["object_permission_skills"] as const; const isParsableJson = (value: string | undefined): boolean => { if (!value) { @@ -214,6 +218,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser const [mcpSettingsOpen, setMcpSettingsOpen] = useState(false); const [agentSettingsOpen, setAgentSettingsOpen] = useState(false); const [searchToolSettingsOpen, setSearchToolSettingsOpen] = useState(false); + const [skillSettingsOpen, setSkillSettingsOpen] = useState(false); const adminOrgs = useMemo( () => getAdminOrganizations(userRole, userID, organizations), @@ -505,6 +510,14 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser delete formValues.object_permission_search_tools; } + if (Array.isArray(formValues.object_permission_skills) && formValues.object_permission_skills.length > 0) { + if (!formValues.object_permission) { + formValues.object_permission = {}; + } + formValues.object_permission.skills = formValues.object_permission_skills; + } + delete formValues.object_permission_skills; + // Add model_aliases if any are defined if (Object.keys(modelAliases).length > 0) { formValues.model_aliases = modelAliases; @@ -540,6 +553,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser ...(mcpSettingsOpen ? [] : MCP_SETTINGS_FIELDS), ...(agentSettingsOpen ? [] : AGENT_SETTINGS_FIELDS), ...(searchToolSettingsOpen ? [] : SEARCH_TOOL_SETTINGS_FIELDS), + ...(skillSettingsOpen ? [] : SKILL_SETTINGS_FIELDS), ]); return Object.fromEntries(Object.entries(values).filter(([key]) => !unmounted.has(key))); }; @@ -1163,6 +1177,38 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser + + + Skill Settings + + + + + {({ value, onChange }) => ( + + )} + + + + Logging Settings diff --git a/ui/litellm-dashboard/src/components/object_permissions_view.tsx b/ui/litellm-dashboard/src/components/object_permissions_view.tsx index e90645a9e9f..ae46729222a 100644 --- a/ui/litellm-dashboard/src/components/object_permissions_view.tsx +++ b/ui/litellm-dashboard/src/components/object_permissions_view.tsx @@ -30,6 +30,7 @@ export function ObjectPermissionsView({ const agents = objectPermission?.agents || []; const agentAccessGroups = objectPermission?.agent_access_groups || []; const searchTools = objectPermission?.search_tools || []; + const skills = objectPermission?.skills || []; const content = (
@@ -58,6 +59,16 @@ export function ObjectPermissionsView({

{searchTools.join(", ")}

)}
+
+

Skills

+ {skills.length === 0 ? ( +

+ No private skills granted. Only enabled (public) Claude Code plugins are visible. +

+ ) : ( +

{skills.join(", ")}

+ )} +
); diff --git a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts index 8b7283370a2..fef94cc3c2b 100644 --- a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts +++ b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts @@ -301,6 +301,16 @@ describe("object_permission", () => { ).toStrictEqual(aliasOnly({ object_permission: { agents: ["a-1"], agent_access_groups: ["ag-1"] } })); }); + it("moves the selected skills under object_permission and off the top level", () => { + expect(payloadOf(build({ key_alias: "my-key", allowed_skills: ["private-skill"] }))).toStrictEqual( + aliasOnly({ object_permission: { skills: ["private-skill"] } }), + ); + }); + + it("sends no object_permission for an empty skill selection", () => { + expect(payloadOf(build({ key_alias: "my-key", allowed_skills: [] }))).toStrictEqual(aliasOnly()); + }); + it("merges every source into a single object_permission", () => { const everySource = { key_alias: "my-key", @@ -308,6 +318,7 @@ describe("object_permission", () => { allowed_mcp_servers_and_groups: { servers: ["s-1"], accessGroups: ["g-1"], toolsets: ["t-1"] }, mcp_tool_permissions: { "s-1": ["read"] }, allowed_agents_and_groups: { agents: ["a-1"], accessGroups: ["ag-1"] }, + allowed_skills: ["private-skill"], }; expect(payloadOf(build(everySource))).toStrictEqual( aliasOnly({ @@ -319,6 +330,7 @@ describe("object_permission", () => { mcp_tool_permissions: { "s-1": ["read"] }, agents: ["a-1"], agent_access_groups: ["ag-1"], + skills: ["private-skill"], }, }), ); diff --git a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts index a528dbaeb2c..37a973d5def 100644 --- a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts +++ b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.ts @@ -112,6 +112,7 @@ interface PermissionSources { readonly toolPermissions: unknown | undefined; readonly extraMcpAccessGroups: unknown[] | undefined; readonly agents: AgentSelection | undefined; + readonly skills: unknown[] | undefined; } const readPermissionSources = (values: Record): PermissionSources => ({ @@ -120,6 +121,7 @@ const readPermissionSources = (values: Record): PermissionSourc toolPermissions: readToolPermissions(values.mcp_tool_permissions), extraMcpAccessGroups: nonEmptyList(values.allowed_mcp_access_groups), agents: readAgentSelection(values.allowed_agents_and_groups), + skills: nonEmptyList(values.allowed_skills), }); const buildObjectPermission = ({ @@ -128,6 +130,7 @@ const buildObjectPermission = ({ toolPermissions, extraMcpAccessGroups, agents, + skills, }: PermissionSources): Record | undefined => { const permission: Record = { ...(vectorStores && { vector_stores: vectorStores }), @@ -138,6 +141,7 @@ const buildObjectPermission = ({ ...(extraMcpAccessGroups && { mcp_access_groups: extraMcpAccessGroups }), ...(agents?.agents && { agents: agents.agents }), ...(agents?.accessGroups && { agent_access_groups: agents.accessGroups }), + ...(skills && { skills }), }; return Object.keys(permission).length > 0 ? permission : undefined; }; @@ -148,6 +152,7 @@ const consumedSourceKeys = ( ): ReadonlySet => new Set([ "mcp_tool_permissions", + "allowed_skills", ...(values.disable_global_guardrails ? [] : ["disable_global_guardrails"]), ...(vectorStores ? ["allowed_vector_store_ids"] : []), ...(mcp ? ["allowed_mcp_servers_and_groups"] : []), diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index 3471ef00eb0..2bf39bf4cd7 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -84,6 +84,13 @@ vi.mock("../networking", async (importOriginal) => { getPossibleUserRoles: vi.fn().mockResolvedValue({}), userFilterUICall: vi.fn().mockResolvedValue([]), getAgentsList: vi.fn().mockResolvedValue({ agents: [] }), + getClaudeCodePluginsList: vi.fn().mockResolvedValue({ + plugins: [ + { name: "public-skill", enabled: true }, + { name: "private-skill", enabled: false }, + ], + count: 2, + }), getPassThroughEndpointsCall: vi.fn().mockResolvedValue({ endpoints: [] }), vectorStoreListCall: vi.fn().mockResolvedValue({ data: [] }), listMCPTools: vi.fn().mockResolvedValue(emptyMcpTools), @@ -109,6 +116,7 @@ const OPENAPI_SCHEMA = { const SECTIONS = { mcp: /MCP Settings/i, agent: /Agent Settings/i, + skill: /Skill Settings/i, logging: /Logging Settings/i, router: /Router Settings/i, aliases: /Model Aliases/i, @@ -164,6 +172,7 @@ const ROUTER_SETTINGS_DEFAULT = { const SECTION_PAYLOAD_ADDITIONS: Record> = { mcp: { allowed_mcp_servers_and_groups: { servers: [], accessGroups: [] } }, agent: { allowed_agents_and_groups: undefined }, + skill: {}, logging: {}, router: { router_settings: ROUTER_SETTINGS_DEFAULT }, aliases: {}, @@ -329,6 +338,21 @@ describe("CreateKey", () => { expect(Object.keys(serialised).sort()).toStrictEqual([...wireKeys].sort()); }); + it("moves a picked private skill under object_permission.skills and off the top level", async () => { + await openModal(); + await nameTheKey(); + await openSection(/Optional Settings/i); + await openSection(SECTIONS.skill); + await userEvent.click(await screen.findByRole("combobox", { name: "Select skills (optional)" })); + await userEvent.click(await screen.findByRole("option", { name: "private-skill (private)" })); + await userEvent.keyboard("{Escape}"); + await submit(); + + const payload = await createdPayload(); + expect(payload.object_permission).toStrictEqual({ skills: ["private-skill"] }); + expect(payload).not.toHaveProperty("allowed_skills"); + }); + it("omits a budget typed into a section the user closed again, rather than sending it as null", async () => { await openModal(); await nameTheKey(); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 16ea99ea94c..831cb0cf6a6 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -27,6 +27,7 @@ import React, { useEffect, useMemo, useRef, useState } from "react"; import { type Control, useForm, useWatch, type UseFormSetValue } from "react-hook-form"; import { rolesWithWriteAccess } from "../../utils/roles"; import AgentSelector from "../agent_management/AgentSelector"; +import SkillSelector from "../skills/SkillSelector"; import AccessGroupSelector from "../common_components/AccessGroupSelector"; import BudgetDurationDropdown from "../common_components/budget_duration_dropdown"; import SchemaFormFields from "../common_components/check_openapi_schema"; @@ -1557,6 +1558,36 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp + + + Skill Settings + + + + + Allowed Skills{" "} + + + + + } + name="allowed_skills" + help="Select private skills this key can access in the Claude Code marketplace" + > + {(control) => ( + + )} + + + + {premiumUser ? ( diff --git a/ui/litellm-dashboard/src/components/skills/SkillSelector.test.tsx b/ui/litellm-dashboard/src/components/skills/SkillSelector.test.tsx new file mode 100644 index 00000000000..59aff7c25de --- /dev/null +++ b/ui/litellm-dashboard/src/components/skills/SkillSelector.test.tsx @@ -0,0 +1,54 @@ +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { getClaudeCodePluginsList } from "../networking"; +import SkillSelector from "./SkillSelector"; + +vi.mock("../networking", () => ({ + getClaudeCodePluginsList: vi.fn(), +})); + +describe("SkillSelector", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getClaudeCodePluginsList).mockResolvedValue({ + plugins: [ + { name: "public-skill", enabled: true }, + { name: "private-skill", enabled: false }, + ], + count: 2, + }); + }); + + it("should list marketplace plugins and mark disabled ones as private", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("combobox")); + + expect(await screen.findByRole("option", { name: "public-skill" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "private-skill (private)" })).toBeInTheDocument(); + expect(getClaudeCodePluginsList).toHaveBeenCalledWith("token"); + }); + + it("should report the selected skill names", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + renderWithProviders(); + + await user.click(screen.getByRole("combobox")); + await user.click(await screen.findByRole("option", { name: "private-skill (private)" })); + + expect(onChange).toHaveBeenCalledWith(["private-skill"]); + }); + + it("should clear all selected skills", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: "Clear all skills" })); + + expect(onChange).toHaveBeenCalledWith([]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/skills/SkillSelector.tsx b/ui/litellm-dashboard/src/components/skills/SkillSelector.tsx new file mode 100644 index 00000000000..b21c8b7e872 --- /dev/null +++ b/ui/litellm-dashboard/src/components/skills/SkillSelector.tsx @@ -0,0 +1,109 @@ +import React, { useEffect, useState } from "react"; +import { + Combobox, + ComboboxChip, + ComboboxChips, + ComboboxChipsInput, + ComboboxClear, + ComboboxContent, + ComboboxEmpty, + ComboboxItem, + ComboboxList, + ComboboxValue, + useComboboxAnchor, +} from "@/components/ui/combobox"; +import { cn } from "@/lib/cva.config"; +import { getClaudeCodePluginsList } from "../networking"; + +export interface SkillSelectorProps { + onChange: (selected: string[]) => void; + value?: string[]; + className?: string; + accessToken: string; + placeholder?: string; + disabled?: boolean; +} + +interface SkillOption { + readonly name: string; + readonly enabled: boolean; +} + +const readSkillOptions = (data: unknown): SkillOption[] => { + const plugins = (data as { plugins?: unknown[] } | undefined)?.plugins; + if (!Array.isArray(plugins)) return []; + return plugins.flatMap((plugin) => { + const record = plugin as { name?: unknown; enabled?: unknown }; + return typeof record.name === "string" && record.name.length > 0 + ? [{ name: record.name, enabled: record.enabled !== false }] + : []; + }); +}; + +const SkillSelector: React.FC = ({ + onChange, + value, + className, + accessToken, + placeholder = "Select skills (optional)", + disabled = false, +}) => { + const anchor = useComboboxAnchor(); + const [options, setOptions] = useState([]); + const [loading, setLoading] = useState(false); + + useEffect(() => { + const load = async () => { + if (!accessToken) return; + setLoading(true); + try { + setOptions(readSkillOptions(await getClaudeCodePluginsList(accessToken))); + } catch (e) { + console.error("Failed to load skills:", e); + } finally { + setLoading(false); + } + }; + load(); + }, [accessToken]); + + return ( + option.name)} + value={value ?? []} + onValueChange={(selected: string[]) => onChange(selected)} + disabled={disabled} + > + } className={cn("w-full", className)} aria-busy={loading}> + + {(selected: string[]) => + selected.map((skill) => ( + + {skill} + + )) + } + + + {value && value.length > 0 && } + + + {loading ? "Loading skills…" : "No skills found"} + + {(skill: string) => { + const isPrivate = options.some((option) => option.name === skill && !option.enabled); + return ( + + {skill} + {isPrivate && private} + + ); + }} + + + + ); +}; + +export default SkillSelector; diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index b286c8c1303..a25ca28651a 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -52,6 +52,7 @@ vi.mock("@/components/networking", () => ({ listMCPTools: vi.fn().mockResolvedValue({ tools: [] }), vectorStoreListCall: vi.fn().mockResolvedValue({ data: [] }), getAgentsList: vi.fn().mockResolvedValue({ agents: [] }), + getClaudeCodePluginsList: vi.fn().mockResolvedValue({ plugins: [], count: 0 }), })); const can = vi.fn(); @@ -1990,6 +1991,7 @@ describe("TeamInfoView - the exact bytes the update call sends", () => { agents: [], agent_access_groups: [], vector_stores: ["vs-1"], + skills: [], }; it("leaves every team member key out of the request body for an untouched save with both sections closed", async () => { @@ -2052,6 +2054,47 @@ describe("TeamInfoView - the exact bytes the update call sends", () => { expect(objectPermission.agent_access_groups).toStrictEqual([]); }); + it("resends the stored skills when the selector is left untouched", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ models: ["gpt-4"], object_permission: { skills: ["private-skill"] } }), + ); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await waitFor(() => expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0)); + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await screen.findByLabelText("Team Name"); + + const payload = await save(user); + + const objectPermission = wireBody(payload).object_permission as Record; + expect(objectPermission.skills).toStrictEqual(["private-skill"]); + }); + + it("sends an empty skills array after the last skill chip is removed", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ models: ["gpt-4"], object_permission: { skills: ["private-skill"] } }), + ); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + + renderWithProviders(); + await waitFor(() => expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0)); + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await screen.findByLabelText("Team Name"); + + await user.click(within(screen.getByLabelText("private-skill")).getByRole("button")); + expect(screen.queryByLabelText("private-skill")).not.toBeInTheDocument(); + + const payload = await save(user); + + const objectPermission = wireBody(payload).object_permission as Record; + expect(objectPermission.skills).toStrictEqual([]); + }); + it("sends an empty vector_stores array after the last vector store chip is removed", async () => { const user = userEvent.setup({ delay: null }); await openEditor(user); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 4947cec1441..988e2e082aa 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -89,6 +89,7 @@ import ObjectPermissionsView from "../object_permissions_view"; import NumericalInput from "../shared/numerical_input"; import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; import SearchToolSelector from "../search_tools/SearchToolSelector"; +import SkillSelector from "../skills/SkillSelector"; import EditLoggingSettings from "./EditLoggingSettings"; import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion"; import MemberModal from "./EditMembership"; @@ -371,6 +372,7 @@ const teamUpdateFieldsSchema = z.object({ mcp_tool_permissions: z.record(z.string(), z.array(z.string())).optional(), agents_and_groups: z.object({ agents: z.array(z.string()), accessGroups: z.array(z.string()) }).optional(), object_permission_search_tools: z.array(z.string()).optional(), + object_permission_skills: z.array(z.string()).optional(), organization_id: z.string().nullish(), logging_settings: z.array(z.unknown()).optional(), secret_manager_settings: z.string().optional(), @@ -419,6 +421,7 @@ const EMPTY_TEAM_UPDATE_VALUES: TeamUpdateFormValues = { mcp_tool_permissions: {}, agents_and_groups: { agents: [], accessGroups: [] }, object_permission_search_tools: [], + object_permission_skills: [], organization_id: null, logging_settings: [], secret_manager_settings: "", @@ -485,6 +488,7 @@ const toTeamFormValues = (info: TeamInfoRecord, effectiveGuardrails: string[]): accessGroups: info.object_permission?.agent_access_groups || [], }, object_permission_search_tools: info.object_permission?.search_tools || [], + object_permission_skills: info.object_permission?.skills || [], organization_id: info.organization_id, logging_settings: info.metadata?.logging || [], secret_manager_settings: info.metadata?.secret_manager_settings @@ -1047,6 +1051,10 @@ const TeamInfoView: React.FC = ({ updateData.object_permission.search_tools = values.object_permission_search_tools; } + if (Array.isArray(values.object_permission_skills)) { + updateData.object_permission.skills = values.object_permission_skills; + } + // Pass access_group_ids to the update request if (values.access_group_ids !== undefined) { updateData.access_group_ids = values.access_group_ids; @@ -1848,6 +1856,24 @@ const TeamInfoView: React.FC = ({ + + {({ value, onChange }) => ( + + )} + + {({ id, value, onChange }) => ( ( <> @@ -53,6 +55,36 @@ export const KeyTypeSelect = ({ ); +const SKILLS_HINT = + "Enabled skills are visible to every key. Grant disabled (private) Claude Code plugins to this key here."; + +export const KeyAgentAndSkillFields = ({ + control, + accessToken, +}: { + control: Control; + accessToken: string; +}) => ( + <> + + {({ value, onChange }) => ( + + )} + + + + {({ value, onChange }) => ( + + )} + + +); + export const KeyBudgetNumberField = ({ control, name, diff --git a/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx b/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx index 8a42ecff50c..522af5a85ad 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx @@ -366,6 +366,45 @@ describe("KeyInfoView handleKeyUpdate mcp_toolsets", () => { }); }); +describe("KeyInfoView handleKeyUpdate skills", () => { + it("should forward the skills the edit form supplies into object_permission and drop the form key", async () => { + renderView(true); + + fireEvent.click(screen.getByText("Settings")); + fireEvent.click(screen.getByText("Edit Settings")); + (globalThis as any).__TEST_FORM_VALUES = { + token: "tok_123", + skills: ["private-skill"], + }; + + fireEvent.click(screen.getByText("Mock Submit")); + + await waitFor(() => expect(keyUpdateCallMock).toHaveBeenCalled()); + + const [, sentPayload] = keyUpdateCallMock.mock.calls[0]; + expect(sentPayload.object_permission.skills).toEqual(["private-skill"]); + expect(sentPayload).not.toHaveProperty("skills"); + }); + + it("should send an explicit empty skills list when the form clears every skill", async () => { + renderView(true); + + fireEvent.click(screen.getByText("Settings")); + fireEvent.click(screen.getByText("Edit Settings")); + (globalThis as any).__TEST_FORM_VALUES = { + token: "tok_123", + skills: [], + }; + + fireEvent.click(screen.getByText("Mock Submit")); + + await waitFor(() => expect(keyUpdateCallMock).toHaveBeenCalled()); + + const [, sentPayload] = keyUpdateCallMock.mock.calls[0]; + expect(sentPayload.object_permission.skills).toEqual([]); + }); +}); + describe("KeyInfoView handleKeyUpdate budget_duration", () => { it("should send a canonical budget_duration through unchanged", async () => { renderView(true); diff --git a/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts b/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts index f24fb6e5a86..fec8e749143 100644 --- a/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts +++ b/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts @@ -46,6 +46,7 @@ export interface KeyEditFormValues { mcp_servers_and_groups?: McpServersAndGroups; mcp_tool_permissions?: Record; agents_and_groups?: AgentsAndGroups; + skills?: string[]; organization_id?: string | null; team_id?: string | null; logging_settings?: unknown[]; @@ -102,6 +103,7 @@ export const toKeyEditFormValues = (keyData: KeyResponse): KeyEditFormValues => agents: keyData.object_permission?.agents || [], accessGroups: keyData.object_permission?.agent_access_groups || [], }, + skills: keyData.object_permission?.skills || [], organization_id: keyData.organization_id, team_id: keyData.team_id, logging_settings: extractLoggingSettings(keyData.metadata), @@ -148,6 +150,7 @@ export const keyEditFormSchema = z.object({ mcp_servers_and_groups: z.custom(), mcp_tool_permissions: z.custom | undefined>(), agents_and_groups: z.custom(), + skills: z.custom(), organization_id: z.custom(), team_id: z.custom(), logging_settings: z.custom(), @@ -196,6 +199,7 @@ export const toSubmittedValues = ( mcp_servers_and_groups: values.mcp_servers_and_groups, mcp_tool_permissions: values.mcp_tool_permissions, agents_and_groups: values.agents_and_groups, + skills: values.skills, organization_id: values.organization_id, team_id: values.team_id, logging_settings: values.logging_settings, diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index 35e9268e1c2..b2c2a381b42 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -59,6 +59,7 @@ vi.mock("../networking", async () => { agents: [], }), getAgentAccessGroups: vi.fn().mockResolvedValue([]), + getClaudeCodePluginsList: vi.fn().mockResolvedValue({ plugins: [], count: 0 }), }; }); @@ -135,6 +136,14 @@ vi.mock("../agent_management/AgentSelector", () => ({ ), })); +vi.mock("../skills/SkillSelector", () => ({ + default: ({ onChange }: { onChange: (selected: string[]) => void }) => ( + + ), +})); + vi.mock("../common_components/AccessGroupSelector", () => ({ default: ({ value = [], onChange }: { value?: string[]; onChange?: (v: string[]) => void }) => ( { mcp_servers_and_groups: { servers: [], accessGroups: [], toolsets: [] }, mcp_tool_permissions: {}, agents_and_groups: { agents: [], accessGroups: [] }, + skills: [], organization_id: null, team_id: null, logging_settings: [], @@ -2241,6 +2251,36 @@ describe("KeyEditView", () => { expect(onSubmitMock.mock.calls[0][0].agents_and_groups.agents).toEqual(["agent-1"]); }); + it("carries a picked skill into the payload", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + renderForPayload(onSubmitMock); + await screen.findByRole("button", { name: /save changes/i }); + + await userEvent.click(screen.getByRole("button", { name: "pick skill" })); + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + }); + expect(onSubmitMock.mock.calls[0][0].skills).toEqual(["private-skill"]); + }); + + it("preloads the stored skills into the payload when the selector is left untouched", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + renderForPayload(onSubmitMock, { + ...MOCK_KEY_DATA, + object_permission: { ...MOCK_KEY_DATA.object_permission, skills: ["stored-skill"] }, + } as KeyResponse); + await screen.findByRole("button", { name: /save changes/i }); + + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + }); + expect(onSubmitMock.mock.calls[0][0].skills).toEqual(["stored-skill"]); + }); + it("carries an added logging integration into the payload", async () => { const onSubmitMock = vi.fn().mockResolvedValue(undefined); renderForPayload(onSubmitMock); diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index b1bbbeed6f9..e464dfe5008 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -15,7 +15,6 @@ import { FormField } from "@/components/shared/form/FormField"; import React, { useEffect, useRef, useState } from "react"; import { hasCapability } from "../../utils/capabilities"; import { isProxyAdminRole, rolesWithWriteAccess } from "../../utils/roles"; -import AgentSelector from "../agent_management/AgentSelector"; import AccessGroupSelector from "../common_components/AccessGroupSelector"; import BudgetDurationDropdown from "../common_components/budget_duration_dropdown"; import { mapInternalToDisplayNames } from "../callback_info_helpers"; @@ -32,9 +31,8 @@ import { modelSentinelOptions, parseAllowedRoutes, } from "./keyEditFieldNormalizers"; -import { KeyBudgetNumberField, KeyTypeSelect, labelWithHint } from "./KeyEditViewControls"; +import { KeyAgentAndSkillFields, KeyBudgetNumberField, KeyTypeSelect, labelWithHint } from "./KeyEditViewControls"; import { - AgentsAndGroups, KeyEditFormValues, keyEditFormSchema, McpServersAndGroups, @@ -762,16 +760,7 @@ export function KeyEditView({ /> - - {({ value, onChange }) => ( - - )} - + * - claude plugin install @ * + * Without `key` the catalog holds the enabled (public) plugins. With `?key=sk-...` + * the key is authenticated and the catalog also holds the disabled plugins granted + * to it through `object_permission.skills` on the key or its team. + * * Returns: * Marketplace catalog with list of available plugins and their git sources. * * Example: * ```bash * claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json + * claude plugin marketplace add "http://localhost:4000/claude-code/marketplace.json?key=sk-..." * claude plugin install my-plugin@litellm * ``` */ @@ -29333,6 +29338,8 @@ export interface components { models?: string[] | null; /** Search Tools */ search_tools?: string[] | null; + /** Skills */ + skills?: string[] | null; /** Vector Stores */ vector_stores?: string[] | null; }; @@ -29386,6 +29393,8 @@ export interface components { * @default [] */ search_tools: string[] | null; + /** Skills */ + skills?: string[] | null; /** * Vector Stores * @default [] @@ -43206,7 +43215,9 @@ export interface operations { }; get_marketplace_claude_code_marketplace_json_get: { parameters: { - query?: never; + query?: { + key?: string | null; + }; header?: never; path?: never; cookie?: never; @@ -43222,6 +43233,15 @@ export interface operations { "application/json": unknown; }; }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; }; }; list_plugins_claude_code_plugins_get: { From d62493a7794b1963c0c2ab541bda92d6dd896041 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:26:11 -0700 Subject: [PATCH 364/411] test(batch): prove an open Redis breaker keeps enqueued-token reservations quiet --- .../proxy/hooks/test_batch_enqueued_tokens.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py b/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py index edc921a40a3..e3e39a87009 100644 --- a/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py +++ b/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py @@ -7,6 +7,7 @@ response-shape helpers the v3 limiter's post-call hooks rely on. """ import base64 +import logging import socket import uuid from collections.abc import Mapping, Sequence @@ -16,6 +17,7 @@ from typing import Final import pytest from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import BATCH_ENQUEUED_TOKEN_TTL_SECONDS from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.batch_enqueued_tokens import ( @@ -439,3 +441,29 @@ async def test_redis_lua_path_full_lifecycle(): refill = await store.reserve(tokens=50, scopes=(key_scope, team_scope)) assert isinstance(refill, BatchEnqueuedTokenReservation) await store.refund(refill) + + +class _OpenBreakerRedis: + def async_register_script(self, script: str): + async def refused(keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> object: + raise RedisCircuitBreakerOpenError("Redis circuit breaker is open") + + return refused + + +@pytest.mark.asyncio +async def test_an_open_circuit_breaker_keeps_reservations_in_memory_without_a_warning(caplog): + scope = _scope(limit=100) + store = BatchEnqueuedTokenStore( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=_OpenBreakerRedis(), default_in_memory_ttl=60)) # pyright: ignore[reportArgumentType] # duck-typed Redis double + ) + + with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): + reservation = await store.reserve(tokens=60, scopes=(scope,)) + assert isinstance(reservation, BatchEnqueuedTokenReservation) + assert reservation.backend == "memory" + await store.save_reservation("batch_quiet", reservation) + assert await store.pop_reservation("batch_quiet") == reservation + + assert not [record for record in caplog.records if record.levelno >= logging.WARNING] + assert sum("circuit breaker is open" in record.getMessage() for record in caplog.records) == 3 From c7d3a6a1d46d4e11c37df7caaa59f4fb582c4dea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:31:48 -0700 Subject: [PATCH 365/411] refactor(redis): drop the narrative docstrings on the breaker helper and its tests --- litellm/caching/redis_cache.py | 7 +------ tests/test_litellm/caching/test_dual_cache.py | 5 ----- tests/test_litellm/caching/test_redis_cache.py | 5 ----- 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index d938b94df43..6b93529e456 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -393,17 +393,12 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep class RedisCircuitBreakerOpenError(Exception): - """Raised in place of a Redis call while the circuit breaker is open.""" + pass def log_redis_failure( logger: logging.Logger, level: int, message: str, exc: BaseException, with_traceback: bool = False ) -> None: - """Log a Redis failure the caller is about to swallow. - - An open breaker refuses every call until Redis recovers and announced itself once when it - opened, so the calls it refuses are logged at debug instead of once per request at ``level``. - """ if isinstance(exc, RedisCircuitBreakerOpenError): logger.debug("%s: %s", message, exc) return diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index b0c17507adb..eae8bbfdaff 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -581,8 +581,6 @@ async def test_dual_cache_late_attach_redis_wires_writes_and_ttl_async(): class _OpenBreakerRedis: - """A RedisCache whose breaker is open, so every guarded call is refused before it starts.""" - def __init__(self) -> None: from litellm.caching.redis_cache import RedisCircuitBreaker @@ -639,9 +637,6 @@ class _OpenBreakerRedis: ids=["get", "batch_get", "set", "set_pipeline", "increment_pipeline", "increment"], ) async def test_an_open_circuit_breaker_is_not_an_error_per_request(caplog, call): - """While the breaker is open every request is refused by design, and the breaker already - said so once when it opened; logging each refusal as an ERROR traceback was the storm that - pinned every worker's CPU during a Redis latency blip.""" cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=_OpenBreakerRedis()) # pyright: ignore[reportArgumentType] # duck-typed Redis double caplog.clear() diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index b9d965a8942..bcaa58c9c40 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1016,11 +1016,6 @@ async def test_breaker_metrics_track_state_and_failure_class(): def test_sync_guard_counts_a_timeout_as_a_timeout(): - """A sync Redis timeout must wait out the timeout-only min duration exactly like the async guard. - - Recording it as a hard connectivity failure opened the breaker on the fifth slow reply, - which is how a latency blip took the shared cache out for every worker. - """ from redis.exceptions import TimeoutError as RedisTimeoutError from litellm.caching.redis_cache import RedisCircuitBreaker, _run_under_circuit_breaker_sync From ad78a8f8d0a4e6200a98d5ef0ddae252b1e5c154 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 10 Sep 2026 21:35:49 +0000 Subject: [PATCH 366/411] refactor(caching): walk exception causes iteratively for the breaker timeout check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_cache.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index de4dcb5aa27..742c7144784 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -15,7 +15,7 @@ import hashlib import inspect import json import time -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import Awaitable, Callable, Iterator, Sequence from contextvars import ContextVar from dataclasses import dataclass from datetime import timedelta @@ -334,11 +334,22 @@ def _redis_timeout_error_types() -> tuple[type, ...]: return (RedisTimeoutError, TimeoutError) +_MAX_EXCEPTION_CAUSE_DEPTH: Final = 20 + + +def _exception_cause_chain(exc: BaseException) -> Iterator[BaseException]: + current = exc # rebind-ok: advances one link per iteration of the bounded walk + for _ in range(_MAX_EXCEPTION_CAUSE_DEPTH): + yield current + if current.__cause__ is None: + return + current = current.__cause__ + + def _is_redis_timeout_failure(exc: BaseException) -> bool: - """Follows __cause__: a blocking pool wait raises ConnectionError from asyncio.TimeoutError.""" - if isinstance(exc, _redis_timeout_error_types()): - return True - return exc.__cause__ is not None and _is_redis_timeout_failure(exc.__cause__) + """Walks __cause__: a blocking pool wait raises ConnectionError from asyncio.TimeoutError.""" + timeout_types: Final = _redis_timeout_error_types() + return any(isinstance(cause, timeout_types) for cause in _exception_cause_chain(exc)) class _BreakerMetrics: From 29145d13965df6aed5ebefa9e5ba8662321d54e8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:37:39 -0700 Subject: [PATCH 367/411] test(limiter): inject the open-breaker Redis double instead of replacing script attributes --- .../hooks/test_parallel_request_limiter_v3.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index f97d89b34a2..10c0bb88a82 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -6238,17 +6238,28 @@ async def test_post_call_success_hook_leaves_raw_provider_dict_untouched(): assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []} -@pytest.mark.asyncio -async def test_an_open_circuit_breaker_falls_back_to_the_pipeline_without_a_warning(caplog): - from litellm.caching.redis_cache import RedisCircuitBreakerOpenError - from litellm.types.caching import RedisPipelineIncrementOperation +class _OpenBreakerRedis: + def async_register_script(self, script: str): + async def refused(keys, args): + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError - handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + raise RedisCircuitBreakerOpenError("Redis circuit breaker is open") + + return refused + + async def async_increment_pipeline(self, increment_list, **kwargs): + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError - async def refused_script(keys, args): raise RedisCircuitBreakerOpenError("Redis circuit breaker is open") - handler.token_increment_script = refused_script + +@pytest.mark.asyncio +async def test_an_open_circuit_breaker_falls_back_to_the_pipeline_without_a_warning(caplog): + from litellm.types.caching import RedisPipelineIncrementOperation + + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=_OpenBreakerRedis())) # pyright: ignore[reportArgumentType] # duck-typed Redis double + ) with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): await handler.async_increment_tokens_with_ttl_preservation( @@ -6261,14 +6272,9 @@ async def test_an_open_circuit_breaker_falls_back_to_the_pipeline_without_a_warn @pytest.mark.asyncio async def test_an_open_circuit_breaker_reads_the_sliding_window_locally_without_a_warning(caplog): - from litellm.caching.redis_cache import RedisCircuitBreakerOpenError - - handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) - - async def refused_script(keys, args): - raise RedisCircuitBreakerOpenError("Redis circuit breaker is open") - - handler.batch_rate_limiter_script = refused_script + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=_OpenBreakerRedis())) # pyright: ignore[reportArgumentType] # duck-typed Redis double + ) with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): values = await handler._execute_redis_batch_rate_limiter_script( From dbcb10c57d6546f232c31d1c0c73633edac449d0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:42:39 -0700 Subject: [PATCH 368/411] test(store_model_in_db): expect the placeholder model on rejected unknown-model spend logs The e2e suite asserted the pre-fix behavior (the raw client model string on the spend-log row). The fix in fe6f615c7a replaces it with unknown-model, so the assertion follows. --- tests/store_model_in_db_tests/test_openai_error_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/store_model_in_db_tests/test_openai_error_handling.py b/tests/store_model_in_db_tests/test_openai_error_handling.py index d3f38f93bf3..5de7c427d19 100644 --- a/tests/store_model_in_db_tests/test_openai_error_handling.py +++ b/tests/store_model_in_db_tests/test_openai_error_handling.py @@ -193,7 +193,7 @@ async def test_chat_completion_bad_model_with_spend_logs(): # Verify the structure of the log entry assert log_entry["request_id"] == litellm_call_id - assert log_entry["model"] == "non-existent-model" + assert log_entry["model"] == "unknown-model" assert log_entry["model_group"] in ("", "non-existent-model") assert log_entry["spend"] == 0.0 assert log_entry["total_tokens"] == 0 From df8a9c72ab44c9fa5552b1757f36d2fe99e39a7f Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 10 Sep 2026 21:45:46 +0000 Subject: [PATCH 369/411] fix(caching): keep the sync Redis read and the top-level cache wrapper quiet when the breaker is open The sync RedisCache.get_cache ran outside the breaker and logged with a stray positional argument, so every failure printed a logging-module stack dump. Cache.add_cache and its async twins logged a full traceback for the expected open-breaker fast fail Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/caching.py | 8 +++++- litellm/caching/dual_cache.py | 2 ++ litellm/caching/redis_cache.py | 4 ++- tests/test_litellm/caching/test_caching.py | 26 +++++++++++++++++++ tests/test_litellm/caching/test_dual_cache.py | 15 +++++++++++ .../test_litellm/caching/test_redis_cache.py | 23 +++++++++++++++- 6 files changed, 75 insertions(+), 3 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 884d095793c..b5423998401 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -32,7 +32,7 @@ from .dual_cache import DualCache # noqa: F401 from .gcs_cache import GCSCache from .in_memory_cache import InMemoryCache from .qdrant_semantic_cache import QdrantSemanticCache -from .redis_cache import RedisCache +from .redis_cache import RedisCache, RedisCircuitBreakerOpenError from .redis_cluster_cache import RedisClusterCache from .redis_semantic_cache import RedisSemanticCache from .s3_cache import S3Cache @@ -677,6 +677,8 @@ class Cache: return cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs) self.cache.set_cache(cache_key, cached_data, **kwargs) + except RedisCircuitBreakerOpenError as e: + verbose_logger.debug("LiteLLM Cache: skipped add_cache: %s", e) except Exception as e: verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) @@ -696,6 +698,8 @@ class Cache: await dynamic_cache_object.async_set_cache(cache_key, cached_data, **kwargs) else: await self.cache.async_set_cache(cache_key, cached_data, **kwargs) + except RedisCircuitBreakerOpenError as e: + verbose_logger.debug("LiteLLM Cache: skipped add_cache: %s", e) except Exception as e: verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) @@ -875,6 +879,8 @@ class Cache: await dynamic_cache_object.async_set_cache_pipeline(cache_list=cache_list, **kwargs) else: await self.cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs) + except RedisCircuitBreakerOpenError as e: + verbose_logger.debug("LiteLLM Cache: skipped add_cache: %s", e) except Exception as e: verbose_logger.exception("LiteLLM Cache: Excepton add_cache: %s", e) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 98c6da0f02f..6fb918ce748 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -177,6 +177,8 @@ class DualCache(BaseCache): print_verbose(f"get cache: cache result: {result}") return result + except RedisCircuitBreakerOpenError: + return None except Exception: verbose_logger.error(traceback.format_exc()) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 742c7144784..6c19b018c1d 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -1364,6 +1364,7 @@ class RedisCache(BaseCache): except Exception: return ast.literal_eval(decoded) + @_redis_circuit_breaker_guard_sync def get_cache(self, key, parent_otel_span: Span | None = None, **kwargs): try: key = self.check_and_fix_namespace(key=key) @@ -1384,7 +1385,8 @@ class RedisCache(BaseCache): return self._get_cache_logic(cached_response=cached_response) except Exception as e: # NON blocking - notify users Redis is throwing an exception - verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: ", e) + verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: %s", e) + _record_swallowed_redis_failure(self._circuit_breaker, e) def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]: """ diff --git a/tests/test_litellm/caching/test_caching.py b/tests/test_litellm/caching/test_caching.py index 4d0ec0fb677..5d326a9ece0 100644 --- a/tests/test_litellm/caching/test_caching.py +++ b/tests/test_litellm/caching/test_caching.py @@ -252,3 +252,29 @@ def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param): assert baseline != cache.get_cache_key( model="claude-sonnet-4-5", messages=messages, **anthropic_param ) + + +@pytest.mark.asyncio +async def test_async_add_cache_treats_an_open_breaker_as_a_quiet_skip(caplog): + """The top-level write wrapper logs a full ERROR traceback for any failure. An open Redis + breaker fails every write instantly, so under load that wrapper alone was hundreds of + stack formats per second per replica. Unexpected failures must still get the traceback. + """ + from unittest.mock import AsyncMock + + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + + caplog.set_level(logging.DEBUG, logger="LiteLLM") + cache = Cache(type=LiteLLMCacheType.LOCAL) + cache.cache.async_set_cache = AsyncMock(side_effect=RedisCircuitBreakerOpenError("open")) + + await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + + assert [r.levelno for r in caplog.records if "add_cache" in r.getMessage()] == [logging.DEBUG] + cache.cache.async_set_cache.assert_awaited_once() + + cache.cache.async_set_cache = AsyncMock(side_effect=OSError("disk full")) + await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + + errors = [r for r in caplog.records if r.levelno == logging.ERROR] + assert len(errors) == 1 and errors[0].exc_info is not None diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 46dd2687e48..3e21362b3f0 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -638,3 +638,18 @@ async def test_open_breaker_is_a_quiet_cache_miss(dual_cache_with_open_breaker, noisy = [r for r in caplog.records if r.levelno > logging.DEBUG] assert noisy == [], f"an open breaker must be silent per call, got {[r.getMessage() for r in noisy]}" + + +def test_open_breaker_is_a_quiet_cache_miss_on_the_sync_read_path(dual_cache_with_open_breaker, caplog): + """The sync read runs in the request thread pool for /v1/messages and /v1/responses, so it + must short-circuit on an open breaker like the async path instead of dialing Redis per call. + """ + caplog.set_level(logging.DEBUG, logger="LiteLLM") + redis_client = dual_cache_with_open_breaker.redis_cache.redis_client + redis_client.get.side_effect = AssertionError("an open breaker must not touch Redis") + + for _ in range(50): + assert dual_cache_with_open_breaker.get_cache("lit7468") is None + + noisy = [r for r in caplog.records if r.levelno > logging.DEBUG] + assert noisy == [], f"an open breaker must be silent per call, got {[r.getMessage() for r in noisy]}" diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 856f9b49c56..1a884d44db1 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1,4 +1,5 @@ import asyncio +import logging from collections.abc import Iterator from unittest.mock import AsyncMock, MagicMock, patch @@ -646,7 +647,6 @@ def test_sync_batch_get_cache_survives_a_service_callback_that_raises( from concurrent.futures import ThreadPoolExecutor import litellm - from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD cache, service_logger = sync_batch_cache_with_service_logger @@ -1141,3 +1141,24 @@ async def test_recovery_probe_still_closes_the_breaker(): await asyncio.sleep(0.06) assert await _run_under_circuit_breaker(breaker, "probe", recovered) == "ok" assert breaker.is_open() is False + + +def test_sync_get_cache_failure_feeds_the_breaker_and_logs_a_well_formed_record(sync_batch_redis_cache, caplog): + """The sync read used to log with a stray positional arg, so every Redis failure produced a + `--- Logging error ---` stack dump on stderr, and it sat outside the breaker so it kept dialing + Redis on every request even after the async paths had opened it. + """ + from redis.exceptions import ConnectionError as RedisConnectionError + + from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + + caplog.set_level(logging.ERROR, logger="LiteLLM") + sync_batch_redis_cache.redis_client.get.side_effect = RedisConnectionError("refused") + + for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): + assert sync_batch_redis_cache.get_cache("lit7468") is None + + assert sync_batch_redis_cache._circuit_breaker.is_open() is True + assert sync_batch_redis_cache.redis_client.get.call_count == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + assert all("refused" in record.getMessage() for record in caplog.records) + assert len(caplog.records) == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD From 1987e6e290a8344515f595538e85c3ae00e69a96 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:54:46 -0700 Subject: [PATCH 370/411] test(proxy): pass the request to get_marketplace in the archive marketplace test --- .../proxy/anthropic_endpoints/test_claude_code_marketplace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py index a7c2bd7ba20..a585666743f 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py @@ -481,7 +481,7 @@ async def test_archive_source_registers_and_is_served_verbatim_in_marketplace(): assert response.action == "created" assert response.plugin.source == _ARCHIVE_SOURCE - marketplace = json.loads((await get_marketplace()).body) + marketplace = json.loads((await get_marketplace(request=MagicMock())).body) assert marketplace["plugins"] == [{"name": "s3-skill", "source": _ARCHIVE_SOURCE, "version": "1.0.0"}] From 6632e8b74fecaeb569abdd12be43f6870b45c3b5 Mon Sep 17 00:00:00 2001 From: Kishorekarthik P Date: Sat, 1 Aug 2026 12:57:22 +0530 Subject: [PATCH 371/411] fix(proxy): let internal users read request/response for their own spend logs The Logs drawer gets messages/response from GET /spend/logs/ui/{request_id}; the list endpoint omits those heavy columns for every caller, admins included. That detail route was missing from LiteLLMRoutes.spend_tracking_routes, and check_route_access anchors patterns, so /spend/logs/ui never matched it. Every internal_user got a 403 before the handler ran and the UI fell back to the "Request/Response Data Not Available" banner, even on their own requests Adds the route to spend_tracking_routes so internal_user, internal_user_view_only, admin_viewer and org_admin all inherit it, and drops the now-redundant explicit entry from admin_viewer_routes. The handler already authorizes non-admins per row via _assert_user_can_view_request_id, so no handler-side scoping change is needed That helper returned silently when no spend-log row existed, which the detail handler treats as authorized before asking every custom logger for the payload by raw request_id. With retention pruning the row can be gone while the payload is still in cold storage, so opening the route would have let a non-admin read another tenant's prompt out of S3/GCS. A missing row now falls through to the same 403 as a foreign row, which also removes the exists-but-not-yours oracle Fixes #34099 --- litellm/proxy/_types.py | 9 +- .../spend_management_endpoints.py | 12 +- .../proxy/auth/test_route_checks.py | 76 +++++++ .../test_spend_management_endpoints.py | 186 ++++++++++++++++++ 4 files changed, 273 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 23dc8237160..aa585120e5b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -703,6 +703,7 @@ class LiteLLMRoutes(enum.Enum): "/spend/logs", "/spend/logs/v2", "/spend/logs/ui", + "/spend/logs/ui/{request_id}", "/spend/logs/session/ui", "/key/spend/report", "/user/spend/report", @@ -932,10 +933,10 @@ class LiteLLMRoutes(enum.Enum): # PROXY_ADMIN_VIEW_ONLY — the route gate must match). "/customer/list", "/customer/info", - # UI Logs page detail drawer (single + session) and the filter facets. - # The list endpoint `/spend/logs/ui` is covered via - # spend_tracking_routes below. - "/spend/logs/ui/{logId}", + # UI Logs page session detail drawer and the end-user filter facet. + # The list endpoint `/spend/logs/ui` and the single-log detail route + # `/spend/logs/ui/{request_id}` are covered via spend_tracking_routes + # below. "/spend/logs/session/ui", "/management/v1/spend_logs/end_users", "/management/v1/spend_logs/users", diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index f8831ca4152..88cdf37cabb 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -4633,16 +4633,16 @@ async def _assert_user_can_view_request_id( Verify the requesting non-admin user is allowed to view this spend-log row. Allowed when the log belongs to the user directly, or to one of their permitted teams (admin or ``/spend/logs`` permission). - Raises HTTP 403 if not. + Raises HTTP 403 if not, including when no spend-log row exists for the + request_id (e.g. it was pruned by retention), so a missing row can't be + used to read a payload out of cold storage via the detail endpoint. """ row: Final = await _find_spend_log_row(prisma_client, request_id) - if row is None: + + if row is not None and row.user is not None and row.user == user_api_key_dict.user_id: return - if row.user is not None and row.user == user_api_key_dict.user_id: - return - - if row.team_id: + if row is not None and row.team_id: can_view: Final = await _can_team_member_view_log( prisma_client=prisma_client, user_api_key_dict=user_api_key_dict, diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 4f2ff1582dd..38a0e85c1ea 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -2033,6 +2033,82 @@ def test_proxy_admin_viewer_can_access_logs_page_endpoints(route): ) +@pytest.mark.parametrize( + "user_role", + [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY], +) +def test_internal_user_can_access_logs_drawer_detail_route(user_role): + """ + The Logs drawer detail fetch (GET /spend/logs/ui/{request_id}) must pass + route_checks for plain internal users, not just admins — the handler + itself already self-authorizes row ownership via + _assert_user_can_view_request_id. + """ + route = "/spend/logs/ui/abc-request-id" + user_obj = LiteLLM_UserTable( + user_id="internal_user", + user_email="user@example.com", + user_role=user_role.value, + ) + valid_token = UserAPIKeyAuth( + user_id="internal_user", + user_role=user_role.value, + ) + request = MagicMock(spec=Request) + request.query_params = {} + + try: + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=user_role.value, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + except Exception as e: + pytest.fail(f"{user_role.value} should be able to access {route}. Got error: {str(e)}") + + +@pytest.mark.parametrize( + "route_group_name", + [ + "spend_tracking_routes", + "internal_user_routes", + "internal_user_view_only_routes", + "admin_viewer_routes", + "org_admin_allowed_routes", + ], +) +def test_logs_drawer_detail_route_in_every_route_group(route_group_name): + """ + /spend/logs/ui/{request_id} must be reachable through + RouteChecks.check_route_access under each role's own route group, so a + partial revert (removing the route from `spend_tracking_routes` while + leaving `non_proxy_admin_allowed_routes_check` alone) is also caught. + """ + from litellm.proxy._types import LiteLLMRoutes + + allowed_routes = getattr(LiteLLMRoutes, route_group_name).value + assert RouteChecks.check_route_access( + route="/spend/logs/ui/req-34099", allowed_routes=allowed_routes + ) + + +def test_logs_drawer_detail_route_allowed_for_scoped_virtual_key(): + """ + A virtual key scoped to `allowed_routes=["spend_tracking_routes"]` must be + able to reach the Logs drawer detail route. + """ + valid_token = UserAPIKeyAuth( + user_id="scoped_key_user", + allowed_routes=["spend_tracking_routes"], + ) + assert RouteChecks.is_virtual_key_allowed_to_call_route( + route="/spend/logs/ui/req-34099", valid_token=valid_token + ) + + @pytest.mark.parametrize("route", ADMIN_VIEWER_LOGS_PAGE_ROUTES) def test_internal_user_blocked_from_admin_viewer_logs_routes(route): """ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 329a33eb440..53accfd08ee 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -455,6 +455,34 @@ async def test_assert_user_can_view_request_id_rejects_both_users_none(): assert exc_info.value.status_code == 403 +@pytest.mark.asyncio +async def test_assert_user_can_view_request_id_rejects_missing_row(): + """ + A request_id with no spend-log row (e.g. pruned by retention) must not + authorize reading the payload from cold storage; a missing row is not + the same as an owned row. + """ + + class MockSpendLogs: + async def find_unique(self, where, include=None): + return None + + class MockDB: + def __init__(self): + self.litellm_spendlogs = MockSpendLogs() + + class MockPrisma: + def __init__(self): + self.db = MockDB() + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_1") + with pytest.raises(HTTPException) as exc_info: + await spend_management_endpoints._assert_user_can_view_request_id( + MockPrisma(), auth, "req-missing-row" + ) + assert exc_info.value.status_code == 403 + + def test_ui_view_request_response_forbids_non_admin_without_db(client, monkeypatch): """ Without prisma, non-admins cannot be authorized to read request/response @@ -6777,3 +6805,161 @@ async def test_ui_view_spend_logs_search_returns_flat_rows_when_grouping_by_sess assert "next_session_cursor" not in data finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def _fake_prisma_with_owned_spend_log(owner_user_id, messages_json, response_json): + class _Row: + user = owner_user_id + team_id = None + + class _SpendLogs: + async def find_unique(self, where, include=None): + return _Row() + + class _DB: + def __init__(self): + self.litellm_spendlogs = _SpendLogs() + + async def query_raw(self, _sql, *_args): + return [ + { + "messages": messages_json, + "response": response_json, + "proxy_server_request": "{}", + "metadata": "{}", + } + ] + + class _Prisma: + def __init__(self): + self.db = _DB() + + return _Prisma() + + +def test_ui_view_request_response_internal_user_owner_gets_payload(client, monkeypatch): + """ + An internal_user who owns the spend-log row can fetch the Logs drawer + detail payload for their own request (regression for #34099, where the + route was blocked for INTERNAL_USER before reaching this ownership check). + """ + messages_json = json.dumps([{"role": "user", "content": "hi"}]) + response_json = json.dumps({"choices": [{"message": {"content": "hello"}}]}) + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + _fake_prisma_with_owned_spend_log("user_a", messages_json, response_json), + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_a" + ) + try: + response = client.get( + "/spend/logs/ui/req-owned-by-user-a", + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + body = response.json() + assert json.loads(body["messages"]) == [{"role": "user", "content": "hi"}] + assert json.loads(body["response"]) == { + "choices": [{"message": {"content": "hello"}}] + } + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +class _RecordingAdditionalLoggingUtils: + """Injectable custom logger that records every request_id it's asked for.""" + + def __init__(self, payload): + self._payload = payload + self.requested_ids = [] + + async def get_request_response_payload(self, request_id, start_time_utc, end_time_utc): + self.requested_ids.append(request_id) + return self._payload + + +def test_ui_view_request_response_internal_user_non_owner_forbidden(client, monkeypatch): + """ + A different internal_user requesting someone else's row is forbidden; + guards against _assert_user_can_view_request_id being skipped in the + detail-drawer handler. Also proves the handler stops before it ever asks + a custom logger or the DB for the payload. + """ + messages_json = json.dumps([{"role": "user", "content": "hi"}]) + response_json = json.dumps({"choices": [{"message": {"content": "hello"}}]}) + fake_prisma = _fake_prisma_with_owned_spend_log("user_a", messages_json, response_json) + original_query_raw = fake_prisma.db.query_raw + query_raw_calls = [] + + async def _spy_query_raw(*args, **kwargs): + query_raw_calls.append((args, kwargs)) + return await original_query_raw(*args, **kwargs) + + fake_prisma.db.query_raw = _spy_query_raw + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", fake_prisma) + + custom_logger = _RecordingAdditionalLoggingUtils({"messages": "should-not-be-returned"}) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_active_additional_logging_utils_from_custom_logger", + lambda: [custom_logger], + ) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_b" + ) + try: + response = client.get( + "/spend/logs/ui/req-owned-by-user-a", + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 403 + assert custom_logger.requested_ids == [] + assert query_raw_calls == [] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_ui_view_request_response_internal_user_missing_row_forbidden(client, monkeypatch): + """ + Regression for the fail-open in _assert_user_can_view_request_id: a + request_id with no spend-log row (e.g. pruned by retention) must be + denied before the handler ever consults a custom logger, otherwise a + non-admin who guesses/obtains a request_id could read another tenant's + payload out of cold storage. Fails if `if row is None: return` is + reintroduced. + """ + + class _SpendLogs: + async def find_unique(self, where, include=None): + return None + + class _DB: + def __init__(self): + self.litellm_spendlogs = _SpendLogs() + + from types import SimpleNamespace + + fake_prisma = SimpleNamespace(db=_DB()) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", fake_prisma) + + custom_logger = _RecordingAdditionalLoggingUtils({"messages": "should-not-be-returned"}) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_active_additional_logging_utils_from_custom_logger", + lambda: [custom_logger], + ) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_a" + ) + try: + response = client.get( + "/spend/logs/ui/req-pruned", + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 403 + assert custom_logger.requested_ids == [] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) From d98522b6f662fec6f600ab94817855cb7ff7edce Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:14:37 +0000 Subject: [PATCH 372/411] feat(proxy): share database connections across workers with an in-container pgbouncer (#39683) * feat(proxy): share database connections across workers with an in-container pgbouncer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): parse pgbouncer options iteratively to satisfy the recursion gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): refuse pgbouncer with token db auth and retry failed pooler restarts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): build pgbouncer 1.25.2 from a pinned source archive and verify pooler replacements The public Wolfi repository only carries pgbouncer 1.24.1-r3, which the image scan rejects (CVE-2026-6664, CVE-2026-6665, CVE-2026-6666, CVE-2025-12819). All three images now compile the checksummed 1.25.2 release in a builder stage. The supervisor now waits for a replacement pooler to listen before treating it as recovered, ends and retries one that never does, and takes the same lock for stop() and spawn so no replacement can be started after shutdown began. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): refuse to start pgbouncer on a loopback port another process already owns Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): count pgbouncer ready only once its own unix socket answers, not any listener on the port Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): refuse pgbouncer older than 1.19, whose unix socket cannot vouch for the tcp port Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(helm,terraform): expose the in-container pgbouncer pool for the componentized gateway Add database.connectionPool to helm/litellm and gateway_connection_pool_* to terraform/litellm/aws so the componentized gateway can receive the LITELLM_PGBOUNCER_* env the classic image already honours. Both reject the pool under IAM or Entra token auth at render/plan time: the pooler holds one static database password for the life of the pod or task. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(gateway): launch the componentized gateway image through a pgbouncer-aware supervisor (#40592) The componentized gateway image started uvicorn directly, so the in-container PgBouncer never ran for it: every worker opened its own Prisma pool to the database. It also passed no keep-alive timeout, so behind a load balancer with a 60s idle timeout uvicorn's 5s default closed idle connections first and the balancer returned 502s on scale-out gateway.launch assembles DATABASE_URL, starts PgBouncer once per pod when LITELLM_PGBOUNCER_ENABLED is set, hands the workers the loopback URL and then runs uvicorn on gateway.main:app with KEEPALIVE_TIMEOUT as --timeout-keep-alive. The image builds PgBouncer 1.25.2 from a checksummed tarball, copies the compiled Rust extension into the /app source tree it imports from (it was only in site-packages, which PYTHONPATH=/app shadows) and asserts the native bridge loads. The app user is added to stats_users so operators can read the PgBouncer console with the application credentials The supervisor returns the pooled URL instead of writing into the mapping it was handed, a database user whose name PgBouncer would split into several stats_users entries is refused before the config is written, and the launcher tests drive main() with an injected serve callable Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(terraform): describe the gateway.launch pooler entrypoint in the aws module README Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): run pgbouncer exit hooks only in the parent and copy the CA into the runtime dir Gunicorn workers inherit the parent's atexit table, so a recycled worker (max_requests) stopped the shared pooler and removed its runtime dir, then hung in the inherited Popen lock. The hooks now no-op unless os.getpid() is the process that started PgBouncer A verified TLS upstream named the operator's CA bundle directly, which is often a 0600 root-owned file that nobody (the user PgBouncer drops to) cannot read, so every server connection failed with "failed to load CA". The bundle is copied into the runtime dir next to the ini and chowned with it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(terraform): run the gateway through gateway.launch and add the gcp connection-pool variables Cloud Run and ECS overrode the image command with uvicorn gateway.main:app, which skips the supervisor that starts the in-container PgBouncer, so LITELLM_PGBOUNCER_ENABLED was inert on both stacks. Both now exec python -m gateway.launch (under ddtrace-run when USE_DDTRACE is set), and the gcp module gains gateway_connection_pool_enabled / gateway_pool_max_db_connections / gateway_pool_max_client_conn wired to the gateway service only The test_launch password_env fixture now restores DATABASE_URL even when it was unset: monkeypatch.delenv records nothing for an absent var, so main() left postgresql://...@db.internal in the xdist worker's environ and the key-rotation e2e test in the same proxy-infra shard stopped skipping and tried to reach db.internal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(pgbouncer): keep channel_binding and gssencmode off the loopback URL Prisma would demand TLS channel binding from a pooler that only speaks plain TCP on 127.0.0.1. Also pass the request the marketplace test started needing after #40518 landed on top of #40496 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-unit.yml | 1 + Dockerfile | 19 +- docker/Dockerfile.database | 19 +- docker/Dockerfile.non_root | 19 +- gateway/Dockerfile | 28 +- gateway/launch.py | 71 ++ helm/litellm-helm/templates/deployment.yaml | 8 + .../tests/connection_pool_tests.yaml | 61 ++ helm/litellm-helm/values.yaml | 14 + helm/litellm/templates/_helpers.tpl | 17 + .../litellm/templates/gateway/deployment.yaml | 3 + helm/litellm/tests/connection_pool_tests.yaml | 117 ++++ helm/litellm/values.yaml | 20 + litellm/proxy/db/pgbouncer.py | 525 +++++++++++++++ litellm/proxy/proxy_cli.py | 16 + terraform/litellm/aws/README.md | 33 + terraform/litellm/aws/ecs.tf | 14 +- .../aws/tests/connection_pool.tftest.hcl | 123 ++++ terraform/litellm/aws/variables.tf | 39 ++ terraform/litellm/gcp/README.md | 34 + terraform/litellm/gcp/cloudrun.tf | 10 +- .../gcp/tests/connection_pool.tftest.hcl | 135 ++++ terraform/litellm/gcp/variables.tf | 39 ++ tests/test_gateway/test_launch.py | 155 +++++ tests/test_litellm/proxy/db/test_pgbouncer.py | 624 ++++++++++++++++++ .../test_litellm/test_component_entrypoint.py | 72 +- 26 files changed, 2195 insertions(+), 21 deletions(-) create mode 100644 gateway/launch.py create mode 100644 helm/litellm-helm/tests/connection_pool_tests.yaml create mode 100644 helm/litellm/tests/connection_pool_tests.yaml create mode 100644 litellm/proxy/db/pgbouncer.py create mode 100644 terraform/litellm/aws/tests/connection_pool.tftest.hcl create mode 100644 terraform/litellm/gcp/tests/connection_pool.tftest.hcl create mode 100644 tests/test_gateway/test_launch.py create mode 100644 tests/test_litellm/proxy/db/test_pgbouncer.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index cc606339a20..f55c87c2ae5 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -200,6 +200,7 @@ jobs: tests/test_litellm/proxy/types_utils tests/test_litellm/proxy/logging_endpoints tests/test_litellm/proxy/test_*.py + tests/test_gateway workers: 4 reruns: 2 timeout-minutes: 20 diff --git a/Dockerfile b/Dockerfile index 0a92aa9a68c..759dac76795 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,9 +8,25 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 +# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) +ARG PGBOUNCER_VERSION=1.25.2 +ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332 FROM $UV_IMAGE AS uvbin +FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder +ARG PGBOUNCER_VERSION +ARG PGBOUNCER_SHA256 +USER root +RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl +WORKDIR /build +RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ + echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ + tar xzf pgbouncer.tar.gz --strip-components=1 && \ + ./configure --prefix=/usr/local --with-openssl=/usr && \ + make -j"$(nproc)" pgbouncer && \ + install -m 0755 pgbouncer /usr/local/bin/pgbouncer + # Admin UI builder. Pinned to the build platform so the architecture-independent # Next.js static export compiles once natively even in a multi-arch build, # instead of once per target arch under QEMU. @@ -110,7 +126,8 @@ USER root RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile +RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent +COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app ENV PATH="/app/.venv/bin:${PATH}" \ diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index e9ad2849bb2..b0bf935c616 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -8,9 +8,25 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 +# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) +ARG PGBOUNCER_VERSION=1.25.2 +ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332 FROM $UV_IMAGE AS uvbin +FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder +ARG PGBOUNCER_VERSION +ARG PGBOUNCER_SHA256 +USER root +RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl +WORKDIR /build +RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ + echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ + tar xzf pgbouncer.tar.gz --strip-components=1 && \ + ./configure --prefix=/usr/local --with-openssl=/usr && \ + make -j"$(nproc)" pgbouncer && \ + install -m 0755 pgbouncer /usr/local/bin/pgbouncer + # Admin UI builder. Pinned to the build platform so the architecture-independent # Next.js static export compiles once natively even in a multi-arch build, # instead of once per target arch under QEMU. @@ -101,7 +117,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile +RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent +COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app ENV PATH="/app/.venv/bin:${PATH}" \ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index edf20e8bbff..5d729046678 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -7,9 +7,25 @@ ARG PROXY_EXTRAS_SOURCE=published ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 +# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) +ARG PGBOUNCER_VERSION=1.25.2 +ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332 FROM $UV_IMAGE AS uvbin +FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder +ARG PGBOUNCER_VERSION +ARG PGBOUNCER_SHA256 +USER root +RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl +WORKDIR /build +RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ + echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ + tar xzf pgbouncer.tar.gz --strip-components=1 && \ + ./configure --prefix=/usr/local --with-openssl=/usr && \ + make -j"$(nproc)" pgbouncer && \ + install -m 0755 pgbouncer /usr/local/bin/pgbouncer + # Admin UI builder. Pinned to the build platform so the architecture-independent # Next.js static export compiles once natively even in a multi-arch build, # instead of once per target arch under QEMU. @@ -128,8 +144,9 @@ RUN for i in 1 2 3; do \ apk upgrade --no-cache && break || sleep 5; \ done && \ for i in 1 2 3; do \ - apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \ + apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \ done +COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer # Copy only what runtime needs. The application is installed inside the venv; # the rest of the builder's /app is source and build metadata that must not diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 308d70a6b26..33d3791dbba 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,9 +1,25 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a +# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) +ARG PGBOUNCER_VERSION=1.25.2 +ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332 FROM $UV_IMAGE AS uvbin +FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder +ARG PGBOUNCER_VERSION +ARG PGBOUNCER_SHA256 +USER root +RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl +WORKDIR /build +RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ + echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ + tar xzf pgbouncer.tar.gz --strip-components=1 && \ + ./configure --prefix=/usr/local --with-openssl=/usr && \ + make -j"$(nproc)" pgbouncer && \ + install -m 0755 pgbouncer /usr/local/bin/pgbouncer + # ---------- Builder ---------- FROM $LITELLM_BUILD_IMAGE AS builder @@ -61,6 +77,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra bedrock-realtime \ --python python3.13 +# PYTHONPATH=/app makes the source tree shadow the installed package, so the +# compiled Rust extension must live next to the source or it is never imported. +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma @@ -73,7 +93,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \ + apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -90,15 +110,17 @@ ENV HOME=/home/nonroot \ COPY --from=builder --chown=nonroot:nonroot /app /app COPY --from=builder /opt/prisma /opt/prisma +COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ find /app/.venv -type d -path "*/tornado/test" -delete && \ chmod -R a+rX /opt/prisma && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" + python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \ + python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()" USER nonroot EXPOSE 4000/tcp -ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"] +ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh python -m gateway.launch --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"] CMD ["--host", "0.0.0.0", "--port", "4000"] diff --git a/gateway/launch.py b/gateway/launch.py new file mode 100644 index 00000000000..6b28fafdcf6 --- /dev/null +++ b/gateway/launch.py @@ -0,0 +1,71 @@ +"""Gateway supervisor: assemble DATABASE_URL, start the in-container PgBouncer, then run uvicorn. + +``gateway/main.py`` assembles ``DATABASE_URL`` inside every uvicorn worker, which +is fine for a plain Postgres URL but not for the pooler: PgBouncer must be +started exactly once per pod, before the workers fork, and the workers must be +handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in +``DatabaseURLSettings.apply_to_env`` under password auth, so setting it here is +enough for every worker to pick the pooled URL up unchanged. + +Run with: + python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000 +""" + +import os +import sys +from collections.abc import Callable, Mapping, Sequence +from typing import Final + +from uvicorn.main import main as uvicorn_main + +from litellm.proxy.db.db_url_settings import DatabaseURLSettings +from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_in_container_pgbouncer + +GATEWAY_APP: Final = "gateway.main:app" +KEEPALIVE_FLAG: Final = "--timeout-keep-alive" + + +def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]: + """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly.""" + keepalive: Final = environ.get("KEEPALIVE_TIMEOUT") + if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv): + return (GATEWAY_APP, *argv) + return (GATEWAY_APP, *argv, KEEPALIVE_FLAG, keepalive) + + +def pool_database_url( + settings: DatabaseURLSettings, + pgbouncer: PgBouncerSettings, + environ: Mapping[str, str], +) -> str | PgBouncerError | None: + """Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off. + + The upstream URL is whatever ``apply_to_env`` assembled from the discrete + ``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Token auth is + rejected by the pooler itself, since it holds one password for its lifetime. + """ + if not pgbouncer.enabled: + return None + upstream_url: Final = environ.get("DATABASE_URL") + if upstream_url is None: + return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled") + return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth_enabled=settings.token_auth() is not None) + + +def _serve(argv: Sequence[str]) -> None: + uvicorn_main(tuple(argv), prog_name="uvicorn") + + +def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + settings: Final = DatabaseURLSettings.from_env() + settings.apply_to_env() + pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ) + if isinstance(pooled_url, PgBouncerError): + sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") + if pooled_url is not None: + os.environ["DATABASE_URL"] = pooled_url + serve(uvicorn_argv(argv, os.environ)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml index f7c918a6827..834071eb9b2 100644 --- a/helm/litellm-helm/templates/deployment.yaml +++ b/helm/litellm-helm/templates/deployment.yaml @@ -117,6 +117,14 @@ spec: - name: DATABASE_URL_READ_REPLICA value: {{ .Values.db.readReplicaUrl | quote }} {{- end }} + {{- if .Values.db.connectionPool.enabled }} + - name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: {{ .Values.db.connectionPool.maxDbConnections | quote }} + - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: {{ .Values.db.connectionPool.maxClientConn | quote }} + {{- end }} - name: PROXY_MASTER_KEY valueFrom: secretKeyRef: diff --git a/helm/litellm-helm/tests/connection_pool_tests.yaml b/helm/litellm-helm/tests/connection_pool_tests.yaml new file mode 100644 index 00000000000..af23512dafc --- /dev/null +++ b/helm/litellm-helm/tests/connection_pool_tests.yaml @@ -0,0 +1,61 @@ +suite: test in-container connection pool +templates: + - deployment.yaml + - configmap-litellm.yaml +tests: + - it: should not emit pgbouncer env vars by default + template: deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "20" + + - it: should enable the pool with the default sizing when connectionPool.enabled is set + template: deployment.yaml + set: + db.connectionPool.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "20" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: "1000" + + - it: should pass custom sizing through as strings next to the worker count + template: deployment.yaml + set: + numWorkers: 4 + db.connectionPool.enabled: true + db.connectionPool.maxDbConnections: 8 + db.connectionPool.maxClientConn: 400 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "8" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: "400" + - contains: + path: spec.template.spec.containers[0].args + content: "4" diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 907f85de8fc..db596e2d68e 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -355,6 +355,20 @@ db: # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime). readReplicaUrl: "" + # In-container connection pool (PgBouncer, transaction mode) shared by every + # worker in the pod. Without it each --num_workers worker opens its own + # connection_limit connections to Postgres, so a pod's footprint against the + # database's connection ceiling is workers x connection_limit and grows with + # every replica. With it, the pod holds at most maxDbConnections upstream + # connections no matter how many workers run; the workers connect to the pool + # over loopback, with no extra network hop. Migrations still go straight to + # Postgres. Starting profile for numWorkers: 4 is maxDbConnections: 20, so + # a database with a 5000-connection ceiling fits roughly 200 replicas. + connectionPool: + enabled: false + maxDbConnections: 20 + maxClientConn: 1000 + # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster. # The Stackgres Operator must already be installed within the target # Kubernetes cluster. diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index c459512c7b9..4ad3cfe0484 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -360,6 +360,23 @@ harmless no-op for the Job and authoritative for the app pods. {{- end }} {{- end -}} +{{/* +In-container PgBouncer env for the gateway container. Fails at render time under IAM or Entra auth: the pooler holds one static password for the life of the pod. +*/}} +{{- define "litellm.connectionPoolEnv" -}} +{{- if or .Values.database.writer.useIAMAuth .Values.database.writer.useAzureEntraAuth }} +{{- fail "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" }} +{{- end }} +{{- with .Values.database.connectionPool -}} +- name: LITELLM_PGBOUNCER_ENABLED + value: "true" +- name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: {{ required "database.connectionPool.maxDbConnections is required when the pool is enabled" .maxDbConnections | quote }} +- name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: {{ required "database.connectionPool.maxClientConn is required when the pool is enabled" .maxClientConn | quote }} +{{- end }} +{{- end -}} + {{/* PodDisruptionBudget shared by gateway, backend, and ui. diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 9cb6b07e77b..1e9041f0a33 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -61,6 +61,9 @@ spec: - name: NUM_WORKERS value: {{ .Values.gateway.numWorkers | quote }} {{- end }} + {{- if .Values.database.connectionPool.enabled }} + {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsEnv" . | nindent 12 }} {{- end }} diff --git a/helm/litellm/tests/connection_pool_tests.yaml b/helm/litellm/tests/connection_pool_tests.yaml new file mode 100644 index 00000000000..31be7575f3c --- /dev/null +++ b/helm/litellm/tests/connection_pool_tests.yaml @@ -0,0 +1,117 @@ +suite: test in-container connection pool env vars +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml + - backend/configmap.yaml +values: + - ./values/required.yaml +tests: + - it: renders no pool env by default + template: gateway/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + any: true + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + any: true + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + any: true + + - it: enabled pool renders the three pgbouncer vars with the configured sizes + template: gateway/deployment.yaml + set: + gateway.numWorkers: 4 + database.connectionPool.enabled: true + database.connectionPool.maxDbConnections: 8 + database.connectionPool.maxClientConn: 250 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "8" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: "250" + - contains: + path: spec.template.spec.containers[0].env + content: + name: NUM_WORKERS + value: "4" + + - it: enabled pool uses the chart default sizes + template: gateway/deployment.yaml + set: + database.connectionPool.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "20" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: "1000" + + - it: backend never gets the pool env + template: backend/deployment.yaml + set: + database.connectionPool.enabled: true + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + any: true + + - it: pool with IAM auth fails at render time + template: gateway/deployment.yaml + set: + database.connectionPool.enabled: true + database.writer.useIAMAuth: true + asserts: + - failedTemplate: + errorMessage: "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" + + - it: pool with Entra auth fails at render time + template: gateway/deployment.yaml + set: + database.connectionPool.enabled: true + database.writer.useAzureEntraAuth: true + asserts: + - failedTemplate: + errorMessage: "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" + + - it: IAM auth without the pool still renders + template: gateway/deployment.yaml + set: + database.writer.useIAMAuth: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: IAM_TOKEN_DB_AUTH + value: "true" + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + any: true diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index be1e6d43987..45ab5229f38 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -225,6 +225,26 @@ database: usernameKey: username passwordKey: password + # In-container connection pool (PgBouncer, transaction mode) shared by every + # gateway worker in the pod. Without it each of the `gateway.numWorkers` + # workers opens its own Prisma pool straight to Postgres, so a pod's + # footprint against the database's connection ceiling is + # numWorkers x connection_limit and grows with every replica. With it, the + # pod holds at most maxDbConnections upstream connections no matter how many + # workers run; the workers connect to the pool over loopback, with no extra + # network hop. The chart emits LITELLM_PGBOUNCER_ENABLED / + # LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / LITELLM_PGBOUNCER_MAX_CLIENT_CONN on + # the gateway container only: the backend runs a single worker and the + # migrations Job must keep a direct connection. The pool holds a static + # password, so it cannot be combined with `database.writer.useIAMAuth` or + # `useAzureEntraAuth` (rendering fails). Starting profile for + # `gateway.numWorkers: 4` is maxDbConnections: 20, so a database with a + # 5000-connection ceiling fits roughly 200 gateway replicas. + connectionPool: + enabled: false + maxDbConnections: 20 + maxClientConn: 1000 + # Optional Redis. Leave host empty to disable. # # This is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py new file mode 100644 index 00000000000..7792867600c --- /dev/null +++ b/litellm/proxy/db/pgbouncer.py @@ -0,0 +1,525 @@ +"""In-container PgBouncer shared by every proxy worker. + +Each uvicorn worker owns a Prisma query engine with its own pool of +``connection_limit`` server connections, so the connections a pod holds open +against Postgres scale as ``workers * connection_limit`` and a database with a +fixed connection ceiling runs out of room as pods and workers are added. + +When ``LITELLM_PGBOUNCER_ENABLED`` is set, the supervisor process starts one +PgBouncer next to the workers (no extra network hop: it listens on loopback +inside the pod) in transaction pooling mode, points ``DATABASE_URL`` at it +with ``pgbouncer=true`` so Prisma stops using server-side prepared statements, +and keeps it running for the life of the proxy. Every worker's pool then +becomes cheap client connections to PgBouncer while the upstream connection +count is capped at ``LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS`` per pod, no matter +how many workers run. + +Migrations and the schema diff run in the supervisor before the pooler is +started, so they always go straight to Postgres. ``DATABASE_URL_READ_REPLICA`` +is left untouched. + +The pooler holds the database password from startup, so it cannot be combined +with ``IAM_TOKEN_DB_AUTH`` or ``AZURE_POSTGRESQL_AUTH``: those rotate the +password inside every worker on their own schedule, and PgBouncer would keep +authenticating upstream with the expired token. +""" + +from __future__ import annotations + +import atexit +import os +import re +import shlex +import shutil +import socket +import subprocess +import tempfile +import threading +import time +import urllib.parse +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.token_auth import AZURE_POSTGRESQL_AUTH_ENV_VAR, IAM_TOKEN_DB_AUTH_ENV_VAR + +PGBOUNCER_ENV_PREFIX: Final = "LITELLM_PGBOUNCER_" +PGBOUNCER_LISTEN_ADDR: Final = "127.0.0.1" +PGBOUNCER_INI_NAME: Final = "pgbouncer.ini" +PGBOUNCER_USERLIST_NAME: Final = "userlist.txt" +PGBOUNCER_CA_NAME: Final = "server-ca.pem" +PGBOUNCER_RESTART_DELAY_SECONDS: Final = 1.0 +PGBOUNCER_READY_TIMEOUT_SECONDS: Final = 15.0 +PGBOUNCER_STOP_GRACE_SECONDS: Final = 10.0 +PGBOUNCER_UNPRIVILEGED_USER: Final = "nobody" +PGBOUNCER_MIN_VERSION: Final = (1, 19) +PGBOUNCER_VERSION_PATTERN: Final = re.compile(r"PgBouncer (\d+)\.(\d+)") +PGBOUNCER_LIST_DELIMITER_PATTERN: Final = re.compile(r"[,\s]") +PGBOUNCER_TOKEN_AUTH_CONFLICT: Final = ( + f"the in-container pgbouncer cannot be combined with {IAM_TOKEN_DB_AUTH_ENV_VAR} or " + f"{AZURE_POSTGRESQL_AUTH_ENV_VAR}: each worker rotates the database password on its own schedule and the pooler " + "would keep using the expired token upstream. Disable the pooler or use a static database password" +) + +# Prisma's client-side TLS params describe the hop to Postgres, which becomes +# PgBouncer's server side. They move into ``server_tls_*`` and must not stay on +# the loopback URL: the listener speaks plain TCP and Prisma would refuse it +# under ``sslmode=require`` or ``channel_binding=require``. +PRISMA_TLS_PARAM_KEYS: Final[frozenset[str]] = frozenset( + {"sslmode", "sslcert", "sslaccept", "sslidentity", "sslpassword", "channel_binding", "gssencmode"} +) +POOLED_URL_DROPPED_KEYS: Final[frozenset[str]] = PRISMA_TLS_PARAM_KEYS | frozenset(("options", "pgbouncer")) +PGBOUNCER_SSLMODES: Final[frozenset[str]] = frozenset( + {"disable", "allow", "prefer", "require", "verify-ca", "verify-full"} +) + + +class PgBouncerSettings(BaseSettings): + """``LITELLM_PGBOUNCER_*`` env vars, read once in the supervisor.""" + + model_config = SettingsConfigDict( + env_prefix=PGBOUNCER_ENV_PREFIX, case_sensitive=False, extra="ignore", frozen=True + ) + + enabled: bool = False + port: int = Field(default=6432, ge=1, le=65535) + max_db_connections: int = Field(default=20, ge=1) + max_client_conn: int = Field(default=1000, ge=1) + binary: str = "pgbouncer" + + +@dataclass(frozen=True, slots=True) +class PgBouncerPlan: + ini: str + userlist: str + pooled_url: str + ca_source: str | None = None + + +@dataclass(frozen=True, slots=True) +class PgBouncerError: + reason: str + + +def _single_quoted(value: str) -> str: + """Quote for SQL and for PgBouncer's ``[databases]`` connection string: both double a literal ``'``.""" + return "'" + value.replace("'", "''") + "'" + + +def _userlist_quote(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def _option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None: + """The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else. + + Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``; a + detached ``-c`` is folded into the token that follows it first. + """ + folded: Final = tuple( + f"-c{tokens[index + 1]}" if token == "-c" and index + 1 < len(tokens) else token + for index, token in enumerate(tokens) + if index == 0 or tokens[index - 1] != "-c" + ) + settings: Final = tuple(token[2:] for token in folded if token.startswith(("-c", "--")) and "=" in token[2:]) + return settings if len(settings) == len(folded) else None + + +def _connect_query(options: str) -> str | PgBouncerError: + """Turn Prisma's ``options=-c name=value ...`` startup param into ``SET`` statements. + + PgBouncer rejects any ``-c`` setting in ``options`` that is not one of the + handful it tracks (``statement_timeout`` and ``lock_timeout`` are not), so + the settings are applied to each new server connection instead. Every + client shares them, which is what the single ``DATABASE_URL`` gave anyway. + """ + settings: Final = _option_settings(tuple(shlex.split(options))) + if settings is None: + return PgBouncerError(f"cannot translate the DATABASE_URL options {options!r} into PgBouncer settings") + return "; ".join( + f"SET {name.strip()} TO {_single_quoted(value.strip())}" + for name, value in (setting.split("=", 1) for setting in settings) + ) + + +def _server_tls_settings(sslmode: str, sslcert: str, sslaccept: str, ca_path: Path) -> tuple[str, ...] | PgBouncerError: + """``server_tls_*`` lines naming ``ca_path``, the runtime-dir copy of the bundle: the original (or the + 0600 root pinned by ``pin_bundle_root``) is often unreadable for the user PgBouncer drops to.""" + if sslmode not in PGBOUNCER_SSLMODES: + return PgBouncerError(f"unsupported sslmode {sslmode!r} on DATABASE_URL") + verify: Final = sslmode in ("verify-ca", "verify-full") or (sslmode == "require" and sslaccept == "strict") + if verify and not sslcert: + return PgBouncerError( + "DATABASE_URL asks for a verified TLS connection but names no CA bundle; " + "add sslcert= (or sslrootcert=) so the in-container PgBouncer can verify Postgres" + ) + mode: Final = "verify-full" if verify else sslmode + return (f"server_tls_sslmode = {mode}", *((f"server_tls_ca_file = {ca_path}",) if sslcert else ())) + + +def plan_pgbouncer( + upstream_url: str, + settings: PgBouncerSettings, + runtime_dir: Path, + run_as_user: str | None, +) -> PgBouncerPlan | PgBouncerError: + """Render the PgBouncer config for ``upstream_url`` and the loopback URL Prisma uses instead. + + Params describing Prisma's own pool (``connection_limit``, ``pool_timeout``, + ...) stay on the pooled URL; the TLS params and ``options`` describe the hop + to Postgres and move into the PgBouncer config. ``run_as_user`` is the + unprivileged user PgBouncer drops to when the proxy runs as root, which + PgBouncer itself refuses to do. + """ + parsed: Final = urllib.parse.urlsplit(upstream_url) + params: Final[Mapping[str, str]] = MappingProxyType( + dict(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)) + ) + dbname: Final = urllib.parse.unquote(parsed.path.lstrip("/")) + username: Final = urllib.parse.unquote(parsed.username or "") + password: Final = None if parsed.password is None else urllib.parse.unquote(parsed.password) + if not parsed.hostname or not username or password is None or not dbname: + return PgBouncerError( + "DATABASE_URL must carry a host, user, password and database name for the in-container PgBouncer" + ) + if PGBOUNCER_LIST_DELIMITER_PATTERN.search(username): + return PgBouncerError( + f"the database user {username!r} cannot be named in PgBouncer's stats_users list: " + "PgBouncer splits list settings on commas and whitespace and has no quoting for them" + ) + if "sslidentity" in params: + return PgBouncerError("client certificates (sslidentity) are not supported with the in-container PgBouncer") + tls: Final = _server_tls_settings( + params.get("sslmode", "prefer"), + params.get("sslcert", ""), + params.get("sslaccept", ""), + runtime_dir / PGBOUNCER_CA_NAME, + ) + if isinstance(tls, PgBouncerError): + return tls + connect_query: Final = _connect_query(params["options"]) if params.get("options") else "" + if isinstance(connect_query, PgBouncerError): + return connect_query + upstream: Final = " ".join( + ( + f"host={_single_quoted(parsed.hostname)}", + f"port={parsed.port or 5432}", + f"dbname={_single_quoted(dbname)}", + f"user={_single_quoted(username)}", + f"password={_single_quoted(password)}", + *((f"connect_query={_single_quoted(connect_query)}",) if connect_query else ()), + ) + ) + ini: Final = "\n".join( + ( + "[databases]", + f"{dbname} = {upstream}", + "", + "[pgbouncer]", + f"listen_addr = {PGBOUNCER_LISTEN_ADDR}", + f"listen_port = {settings.port}", + f"unix_socket_dir = {runtime_dir}", + f"auth_file = {runtime_dir / PGBOUNCER_USERLIST_NAME}", + "auth_type = scram-sha-256", + f"stats_users = {username}", + "pool_mode = transaction", + f"max_client_conn = {settings.max_client_conn}", + f"default_pool_size = {settings.max_db_connections}", + f"max_db_connections = {settings.max_db_connections}", + "ignore_startup_parameters = extra_float_digits", + *tls, + *((f"user = {run_as_user}",) if run_as_user else ()), + "", + ) + ) + userlist: Final = f"{_userlist_quote(username)} {_userlist_quote(password)}\n" + pooled_query: Final = urllib.parse.urlencode( + (*((key, value) for key, value in params.items() if key not in POOLED_URL_DROPPED_KEYS), ("pgbouncer", "true")) + ) + credentials: Final = f"{urllib.parse.quote(username, safe='')}:{urllib.parse.quote(password, safe='')}" + pooled_url: Final = urllib.parse.urlunsplit( + parsed._replace(netloc=f"{credentials}@{PGBOUNCER_LISTEN_ADDR}:{settings.port}", query=pooled_query) + ) + return PgBouncerPlan(ini=ini, userlist=userlist, pooled_url=pooled_url, ca_source=params.get("sslcert") or None) + + +def write_pgbouncer_files(plan: PgBouncerPlan, runtime_dir: Path, run_as_user: str | None) -> Path | PgBouncerError: + """Write the ini, userlist (both hold the password, so mode 0600) and CA copy, and return the ini path. + + ``run_as_user`` is the user PgBouncer drops to when started as root; it has + to own the files it re-reads on reload and the socket directory. + """ + ini_path: Final = runtime_dir / PGBOUNCER_INI_NAME + userlist_path: Final = runtime_dir / PGBOUNCER_USERLIST_NAME + ca_path: Final = runtime_dir / PGBOUNCER_CA_NAME + if plan.ca_source is not None: + try: + shutil.copyfile(plan.ca_source, ca_path) + except OSError as error: + return PgBouncerError(f"cannot read the CA bundle {plan.ca_source!r} named by sslcert: {error}") + for path, content in ((userlist_path, plan.userlist), (ini_path, plan.ini)): + path.touch(mode=0o600) + path.write_text(content, encoding="utf-8") + if run_as_user is not None: + runtime_dir.chmod(0o700) + for path in (runtime_dir, ini_path, userlist_path, *((ca_path,) if plan.ca_source is not None else ())): + shutil.chown(path, user=run_as_user) + return ini_path + + +def _port_open(port: int) -> bool: + try: + with socket.create_connection((PGBOUNCER_LISTEN_ADDR, port), timeout=0.5): + return True + except OSError: + return False + + +def _unix_socket_open(path: Path) -> bool: + with socket.socket(socket.AF_UNIX) as probe: + probe.settimeout(0.5) + try: + probe.connect(str(path)) + except OSError: + return False + return True + + +def unix_socket_path(runtime_dir: Path, port: int) -> Path: + return runtime_dir / f".s.PGSQL.{port}" + + +def pgbouncer_version(binary: str) -> tuple[int, int] | PgBouncerError: + """``(major, minor)`` from `` --version``. + + Readiness relies on PgBouncer exiting when it cannot bind its TCP port, + which it does from 1.19 on. Older releases log a warning and serve the unix + socket alone, so their socket would vouch for a port held by someone else. + """ + try: + output: Final = subprocess.run( + (binary, "--version"), capture_output=True, text=True, check=False, timeout=10 + ).stdout + except (OSError, subprocess.TimeoutExpired) as run_error: + return PgBouncerError(f"could not run {binary!r} --version: {run_error}") + found: Final = PGBOUNCER_VERSION_PATTERN.search(output) + if found is None: + return PgBouncerError(f"{binary!r} --version did not report a PgBouncer version: {output.strip()!r}") + return int(found[1]), int(found[2]) + + +def _end(process: subprocess.Popen[bytes]) -> None: + if process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=PGBOUNCER_STOP_GRACE_SECONDS) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +class PgBouncerProcess: + """Runs ``argv`` as a foreground child and restarts it whenever it exits on its own. + + Prisma reconnects by itself after a failed query, so a PgBouncer crash + costs the requests in flight plus one failed query per idle pooled + connection the crash severed, and nothing else once the replacement is + listening again. A replacement that cannot be spawned, finds its port + taken, exits again or never starts listening is retried every + ``restart_delay_seconds`` until ``stop`` is called. + + A connect probe of ``port`` cannot tell the child from another process + that grabbed the port after the availability check, so readiness also + needs ``socket_path``: the unix socket PgBouncer creates in the private + runtime directory, which it only does once every TCP listener is bound + (PgBouncer 1.19 or newer, see ``pgbouncer_version``). + """ + + def __init__( + self, + argv: Sequence[str], + port: int, + socket_path: Path, + restart_delay_seconds: float = PGBOUNCER_RESTART_DELAY_SECONDS, + ready_timeout_seconds: float = PGBOUNCER_READY_TIMEOUT_SECONDS, + ) -> None: + self.argv: Final = tuple(argv) + self.port: Final = port + self.socket_path: Final = socket_path + self.restart_delay_seconds: Final = restart_delay_seconds + self.ready_timeout_seconds: Final = ready_timeout_seconds + self._stopping: Final = threading.Event() + self._lock: Final = threading.Lock() + self._process: subprocess.Popen[bytes] | None = None + + @property + def pid(self) -> int | None: + with self._lock: + return None if self._process is None else self._process.pid + + def _spawn(self) -> subprocess.Popen[bytes] | PgBouncerError | None: + """Start a child, or None once ``stop`` ran; both take the lock so no child can slip in after a stop. + + The port has to be free first: a listener that is already there would + pass the readiness check while the child fails to bind. + """ + with self._lock: + if self._stopping.is_set(): + return None + if _port_open(self.port): + return PgBouncerError(f"{PGBOUNCER_LISTEN_ADDR}:{self.port} is already in use by another process") + try: + process: Final = subprocess.Popen(self.argv) + except OSError as spawn_error: + return PgBouncerError(f"could not start {self.argv[0]!r}: {spawn_error}") + self._process = process + return process + + def _wait_ready(self, process: subprocess.Popen[bytes]) -> PgBouncerError | None: + deadline: Final = time.monotonic() + self.ready_timeout_seconds + while time.monotonic() < deadline: + if process.poll() is not None: + return PgBouncerError(f"pgbouncer exited with status {process.returncode} during startup") + if _port_open(self.port) and _unix_socket_open(self.socket_path): + return None + time.sleep(0.1) + if _port_open(self.port): + return PgBouncerError( + f"{PGBOUNCER_LISTEN_ADDR}:{self.port} is served by another process, not the pgbouncer that was started" + ) + return PgBouncerError( + f"pgbouncer did not start listening on {PGBOUNCER_LISTEN_ADDR}:{self.port} " + f"within {self.ready_timeout_seconds:.0f}s" + ) + + def start(self) -> PgBouncerError | None: + """Spawn PgBouncer, wait until it listens on port and unix socket, then supervise it from a daemon thread.""" + process: Final = self._spawn() + if process is None: + return PgBouncerError("pgbouncer was stopped before it started") + if isinstance(process, PgBouncerError): + return process + not_ready: Final = self._wait_ready(process) + if not_ready is not None: + self.stop() + return not_ready + self._watch(process) + return None + + def _watch(self, process: subprocess.Popen[bytes]) -> None: + threading.Thread( + target=self._supervise, args=(process,), daemon=True, name="litellm-pgbouncer-supervisor" + ).start() + + def _supervise(self, process: subprocess.Popen[bytes]) -> None: + status: Final = process.wait() + if self._stopping.is_set(): + return + verbose_proxy_logger.error( + "In-container pgbouncer (pid %s) exited with status %s; restarting in %.1fs.", + process.pid, + status, + self.restart_delay_seconds, + ) + self._restart_after_delay() + + def _restart_after_delay(self) -> None: + time.sleep(self.restart_delay_seconds) + process: Final = self._spawn() + if process is None: + return + if isinstance(process, PgBouncerError): + self._retry_restart(process.reason) + return + not_ready: Final = self._wait_ready(process) + if not_ready is None: + self._watch(process) + return + _end(process) + self._retry_restart(not_ready.reason) + + def _retry_restart(self, reason: str) -> None: + if self._stopping.is_set(): + return + verbose_proxy_logger.error( + "In-container pgbouncer could not be restarted (%s); retrying in %.1fs.", reason, self.restart_delay_seconds + ) + threading.Thread(target=self._restart_after_delay, daemon=True, name="litellm-pgbouncer-supervisor").start() + + def stop(self) -> None: + with self._lock: + self._stopping.set() + process: Final = self._process + if process is not None: + _end(process) + + +def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]: + """An exit hook that does nothing in a forked child, which inherits the parent's ``atexit`` table.""" + owner_pid: Final = os.getpid() + + def run() -> None: + if os.getpid() == owner_pid: + action() + + return run + + +def start_in_container_pgbouncer( + settings: PgBouncerSettings, + upstream_url: str, + token_auth_enabled: bool = False, + register_exit_hook: Callable[[Callable[[], None]], object] = atexit.register, +) -> str | PgBouncerError: + """Start the pooler for ``upstream_url`` and return the loopback URL the workers must use. + + The pooler lives as long as this process: it is stopped from the exit hooks + once the worker manager has returned, and only by the process that started + it (gunicorn forks its workers, so they carry the hooks too). PgBouncer + refuses to run as root, so a root proxy (the default image) has it drop to + ``nobody``. + """ + if token_auth_enabled: + return PgBouncerError(PGBOUNCER_TOKEN_AUTH_CONFLICT) + version: Final = pgbouncer_version(settings.binary) + if isinstance(version, PgBouncerError): + return version + if version < PGBOUNCER_MIN_VERSION: + return PgBouncerError( + f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy " + f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} " + "or newer is required" + ) + runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-")) + register_exit_hook(_only_in_this_process(lambda: shutil.rmtree(runtime_dir, ignore_errors=True))) + run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None + plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user) + if isinstance(plan, PgBouncerError): + return plan + ini_path: Final = write_pgbouncer_files(plan, runtime_dir, run_as_user) + if isinstance(ini_path, PgBouncerError): + return ini_path + pooler: Final = PgBouncerProcess( + argv=(settings.binary, str(ini_path)), + port=settings.port, + socket_path=unix_socket_path(runtime_dir, settings.port), + ) + failed: Final = pooler.start() + if failed is not None: + return failed + register_exit_hook(_only_in_this_process(pooler.stop)) + verbose_proxy_logger.info( + "In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections.", + pooler.pid, + PGBOUNCER_LISTEN_ADDR, + settings.port, + settings.max_db_connections, + ) + return plan.pooled_url diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index e245367b1b4..16d76ff0415 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -18,6 +18,7 @@ from pydantic import BaseModel, ConfigDict import litellm from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY +from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_in_container_pgbouncer from litellm.proxy.db.query_engine_reaper import start_query_engine_reaper if TYPE_CHECKING: @@ -1377,6 +1378,21 @@ def run_server( print( f"Unable to connect to DB. DATABASE_URL found in environment, but prisma package not found." # noqa: F541 ) + pgbouncer_settings: Final = PgBouncerSettings() + upstream_database_url: Final = os.getenv("DATABASE_URL") + if pgbouncer_settings.enabled and upstream_database_url is not None: + pooled_database_url: Final = start_in_container_pgbouncer( + pgbouncer_settings, upstream_database_url, token_auth_enabled=wants_rds_iam or wants_azure_entra + ) + if isinstance(pooled_database_url, PgBouncerError): + print( + f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " + f"could not start: {pooled_database_url.reason}\033[0m", + file=sys.stderr, + flush=True, + ) + sys.exit(1) + os.environ["DATABASE_URL"] = pooled_database_url if port == 4000 and ProxyInitializationHelpers._is_port_in_use(port): port = random.randint(1024, 49152) if prometheus_metrics_port == port: diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 6d3e0269a15..986428151e0 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -258,6 +258,39 @@ gateway_metrics_port = 4001 gateway_metrics_scrape_cidrs = ["10.0.0.0/16"] ``` +### In-container connection pool + +Each of the `gateway_num_workers` uvicorn workers opens its own Prisma pool +straight to Postgres, so one task holds `workers x connection_limit` +connections and the fleet's footprint against the database ceiling grows with +every task. `gateway_connection_pool_enabled` runs a PgBouncer (transaction +mode, loopback) inside the gateway container that all workers share, capping +the task at `gateway_pool_max_db_connections` upstream connections however +many workers it runs. `gateway_pool_max_client_conn` bounds the worker-side +connections the pooler accepts. The module sets +`LITELLM_PGBOUNCER_ENABLED`, `LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS` and +`LITELLM_PGBOUNCER_MAX_CLIENT_CONN` on the gateway container only; the backend +and the migration task keep the direct connection. + +```hcl +create_database = false +database_url = "postgresql://litellm:@db.internal:5432/litellm" +gateway_num_workers = 4 +gateway_connection_pool_enabled = true +gateway_pool_max_db_connections = 20 +gateway_pool_max_client_conn = 1000 +``` + +The pool needs a static database password, so it is only valid with an +existing database via `database_url`. The module-created Aurora authenticates +with rotating IAM tokens (see [Aurora + IAM auth](#aurora--iam-auth)), which +the pooler cannot follow, and `terraform plan` rejects that combination. + +The componentized `gateway_image` starts through `python -m gateway.launch`, +which reads these variables, starts the pooler once per task and hands the +workers its loopback URL; the classic `litellm` image honours them the same +way. + ### Scaling the gateway on requests and tokens By default the gateway service target-tracks CPU (`gateway_cpu_target`) and diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index aa2c3d558e2..149842c14ff 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -213,6 +213,12 @@ locals { # otherwise we keep the image's ENTRYPOINT and only override `command`. gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}" + gateway_pool_env = var.gateway_connection_pool_enabled ? [ + { name = "LITELLM_PGBOUNCER_ENABLED", value = "true" }, + { name = "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS", value = tostring(var.gateway_pool_max_db_connections) }, + { name = "LITELLM_PGBOUNCER_MAX_CLIENT_CONN", value = tostring(var.gateway_pool_max_client_conn) }, + ] : [] + metrics_enabled = var.gateway_metrics_port != null metrics_multiproc_dir = "/tmp/litellm_prometheus_multiproc" metrics_volume = "prometheus-multiproc" @@ -253,7 +259,7 @@ locals { backend_uvicorn_args = "--host 0.0.0.0 --port 4001" - gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac" + gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run python -m gateway.launch ${local.gateway_uvicorn_args};; *) exec python -m gateway.launch ${local.gateway_uvicorn_args};; esac" backend_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args};; *) exec uvicorn backend.main:app ${local.backend_uvicorn_args};; esac" gateway_proxy_overrides = local.proxy_config_enabled ? { @@ -298,6 +304,11 @@ resource "aws_ecs_task_definition" "gateway" { ) error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set." } + + precondition { + condition = !var.gateway_connection_pool_enabled || local.byo_database + error_message = "gateway_connection_pool_enabled requires an existing database via database_url with create_database = false: the module-created Aurora authenticates with IAM tokens, which the in-container pgbouncer cannot follow because it holds a static database password." + } } family = "${local.name}-gateway" @@ -323,6 +334,7 @@ resource "aws_ecs_task_definition" "gateway" { local.gateway_extra_env_list, local.proxy_config_env, local.metrics_env, + local.gateway_pool_env, ) secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) mountPoints = local.metrics_mount_points diff --git a/terraform/litellm/aws/tests/connection_pool.tftest.hcl b/terraform/litellm/aws/tests/connection_pool.tftest.hcl new file mode 100644 index 00000000000..38aa7051134 --- /dev/null +++ b/terraform/litellm/aws/tests/connection_pool.tftest.hcl @@ -0,0 +1,123 @@ +# Plan-only coverage for the in-container PgBouncer knobs on the gateway task. +# `mock_provider` keeps this offline: no AWS credentials, no API calls, no +# resources. Run from terraform/litellm/aws with `terraform test`. + +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} +mock_provider "random" {} + +variables { + region = "us-east-1" + tenant = "acme" + env = "test" + azs = ["us-east-1a", "us-east-1b"] + allow_plaintext_alb = true +} + +run "pool_off_by_default" { + command = plan + + assert { + condition = length(local.gateway_pool_env) == 0 + error_message = "The gateway must get no LITELLM_PGBOUNCER_* env unless gateway_connection_pool_enabled is set." + } +} + +run "pool_enabled_renders_the_three_vars_with_configured_sizes" { + command = plan + + variables { + create_database = false + database_url = "postgresql://litellm:pw@db.internal:5432/litellm" + gateway_num_workers = 4 + gateway_connection_pool_enabled = true + gateway_pool_max_db_connections = 8 + gateway_pool_max_client_conn = 250 + } + + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + local.gateway_pool_env[0].name == "LITELLM_PGBOUNCER_ENABLED" && local.gateway_pool_env[0].value == "true", + local.gateway_pool_env[1].name == "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS" && local.gateway_pool_env[1].value == "8", + local.gateway_pool_env[2].name == "LITELLM_PGBOUNCER_MAX_CLIENT_CONN" && local.gateway_pool_env[2].value == "250", + ]) + error_message = "The pool env must carry the enabled flag and the configured sizes as strings." + } +} + +run "pool_enabled_uses_the_module_default_sizes" { + command = plan + + variables { + create_database = false + database_url = "postgresql://litellm:pw@db.internal:5432/litellm" + gateway_connection_pool_enabled = true + } + + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + local.gateway_pool_env[1].name == "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS" && local.gateway_pool_env[1].value == "20", + local.gateway_pool_env[2].name == "LITELLM_PGBOUNCER_MAX_CLIENT_CONN" && local.gateway_pool_env[2].value == "1000", + ]) + error_message = "The pool env must fall back to the module defaults of 20 upstream and 1000 client connections." + } +} + +run "gateway_starts_through_the_pool_aware_launcher" { + command = plan + + variables { + gateway_num_workers = 4 + } + + assert { + condition = alltrue([ + strcontains(local.gateway_launch_cmd, "exec python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), + strcontains(local.gateway_launch_cmd, "exec ddtrace-run python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), + !strcontains(local.gateway_launch_cmd, "uvicorn gateway.main:app"), + local.gateway_proxy_overrides.command[0] == local.gateway_launch_cmd, + ]) + error_message = "The gateway must start through gateway.launch (with and without ddtrace) so the pooler starts once before uvicorn forks the workers." + } +} + +run "pool_with_module_created_iam_aurora_fails_at_plan" { + command = plan + + variables { + gateway_connection_pool_enabled = true + } + + expect_failures = [ + aws_ecs_task_definition.gateway, + ] +} + +run "pool_without_any_database_fails_at_plan" { + command = plan + + variables { + create_database = false + gateway_connection_pool_enabled = true + } + + expect_failures = [ + aws_ecs_task_definition.gateway, + ] +} + +run "module_created_iam_aurora_without_the_pool_still_plans" { + command = plan + + assert { + condition = contains(local.managed_db_env, { name = "IAM_TOKEN_DB_AUTH", value = "true" }) + error_message = "Without the pool the module-created Aurora must keep IAM token auth." + } +} diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index 10a9b392673..ec8363dbbaf 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -200,6 +200,45 @@ variable "gateway_num_workers" { } } +variable "gateway_connection_pool_enabled" { + description = <<-EOT + Run an in-container PgBouncer (transaction mode, loopback) in each gateway + task, shared by every uvicorn worker. Without it each of the + `gateway_num_workers` workers opens its own Prisma pool straight to + Postgres, so a task's footprint against the database connection ceiling is + workers x connection_limit and grows with every task. Sets + LITELLM_PGBOUNCER_ENABLED / LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / + LITELLM_PGBOUNCER_MAX_CLIENT_CONN on the gateway container only. Requires + an existing database via `database_url`: the module-created Aurora + authenticates with IAM tokens, which the pooler cannot follow because it + holds one static password for the life of the task. + EOT + type = bool + default = false +} + +variable "gateway_pool_max_db_connections" { + description = "Upstream Postgres connections one gateway task may hold when gateway_connection_pool_enabled is set, regardless of gateway_num_workers. 20 suits 4 workers; a 5000-connection database then fits roughly 200 tasks." + type = number + default = 20 + + validation { + condition = var.gateway_pool_max_db_connections >= 1 + error_message = "gateway_pool_max_db_connections must be >= 1." + } +} + +variable "gateway_pool_max_client_conn" { + description = "Client connections the in-container PgBouncer accepts from the gateway workers when gateway_connection_pool_enabled is set." + type = number + default = 1000 + + validation { + condition = var.gateway_pool_max_client_conn >= 1 + error_message = "gateway_pool_max_client_conn must be >= 1." + } +} + variable "backend_cpu" { description = "Fargate CPU units for the backend task (1024 = 1 vCPU)." type = number diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 66095ec0108..9c71d3e15b7 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -287,6 +287,40 @@ the gateway on GKE with the Helm chart's `targetTokensPerSecond` (see "Dependencies only" below) rather than wiring the counter into Cloud Monitoring, which the autoscaler would ignore +### In-container connection pool + +Each of the `gateway_num_workers` uvicorn workers opens its own Prisma pool +straight to Cloud SQL, so one instance holds `workers x connection_limit` +connections and the fleet's footprint against the database ceiling grows with +every instance Cloud Run adds. `gateway_connection_pool_enabled` runs a +PgBouncer (transaction mode, loopback) inside the gateway container that all +workers share, capping the instance at `gateway_pool_max_db_connections` +upstream connections however many workers it runs. +`gateway_pool_max_client_conn` bounds the worker-side connections the pooler +accepts. The module sets `LITELLM_PGBOUNCER_ENABLED`, +`LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS` and `LITELLM_PGBOUNCER_MAX_CLIENT_CONN` +on the gateway service only; the backend service and the migrations job keep +the direct connection + +```hcl +gateway_num_workers = 4 +gateway_connection_pool_enabled = true +gateway_pool_max_db_connections = 20 +gateway_pool_max_client_conn = 1000 +``` + +The pooler holds one static database password for the life of the instance. +This stack authenticates to Cloud SQL with the Secret Manager password (see +[Database authentication](#database-authentication)), so nothing else is +needed; a Cloud SQL Auth Proxy sidecar with IAM auth would not work with the +pool + +The gateway container starts through `python -m gateway.launch` (the +componentized image's own entrypoint) rather than `uvicorn` directly. The +launcher reads these variables, starts the pooler once per instance before +uvicorn forks the workers and hands them its loopback `DATABASE_URL`. It also +honours `KEEPALIVE_TIMEOUT` from `gateway_extra_env` the way the image does + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index 405893b132b..78d4ffa2152 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -138,10 +138,16 @@ locals { "export DATABASE_URL_READ_REPLICA=\"postgresql://$${DATABASE_USER}:$${DATABASE_PASSWORD}@$${DATABASE_HOST_READ_REPLICA}:$${DATABASE_PORT_READ_REPLICA}/$${DATABASE_NAME}\"", ] + gateway_pool_env = var.gateway_connection_pool_enabled ? [ + { name = "LITELLM_PGBOUNCER_ENABLED", value = "true" }, + { name = "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS", value = tostring(var.gateway_pool_max_db_connections) }, + { name = "LITELLM_PGBOUNCER_MAX_CLIENT_CONN", value = tostring(var.gateway_pool_max_client_conn) }, + ] : [] + gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}" backend_uvicorn_args = "--host 0.0.0.0 --port 4001" - gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac" + gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run python -m gateway.launch ${local.gateway_uvicorn_args};; *) exec python -m gateway.launch ${local.gateway_uvicorn_args};; esac" backend_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args};; *) exec uvicorn backend.main:app ${local.backend_uvicorn_args};; esac" gateway_args = join(" && ", concat( @@ -229,7 +235,7 @@ resource "google_cloud_run_v2_service" "gateway" { } dynamic "env" { - for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env, local.metrics_env_kv) + for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env, local.metrics_env_kv, local.gateway_pool_env) content { name = env.value.name value = env.value.value diff --git a/terraform/litellm/gcp/tests/connection_pool.tftest.hcl b/terraform/litellm/gcp/tests/connection_pool.tftest.hcl new file mode 100644 index 00000000000..439fe6b1d71 --- /dev/null +++ b/terraform/litellm/gcp/tests/connection_pool.tftest.hcl @@ -0,0 +1,135 @@ +# Plan-only coverage for the in-container PgBouncer knobs on the gateway +# service. `mock_provider` keeps this offline: no GCP credentials, no API +# calls, no resources. Run from terraform/litellm/gcp with `terraform test`. + +mock_provider "google" { + mock_resource "google_redis_instance" { + defaults = { + host = "10.0.0.4" + port = 6379 + server_ca_certs = [{ + cert = "-----BEGIN CERTIFICATE-----\nmock\n-----END CERTIFICATE-----" + }] + } + } +} + +mock_provider "google-beta" {} +mock_provider "random" {} + +variables { + project_id = "test-project" + tenant = "tenant" + env = "test" + allow_plaintext_lb = true + image_registry = "us-central1-docker.pkg.dev/test-project/litellm" +} + +run "pool_off_by_default" { + command = plan + + assert { + condition = length(local.gateway_pool_env) == 0 + error_message = "The gateway must get no LITELLM_PGBOUNCER_* env unless gateway_connection_pool_enabled is set." + } + + assert { + condition = !anytrue([ + for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : startswith(e.name, "LITELLM_PGBOUNCER_") + ]) + error_message = "The gateway service must carry no LITELLM_PGBOUNCER_* env by default." + } +} + +run "pool_enabled_renders_the_three_vars_with_configured_sizes" { + command = plan + + variables { + gateway_num_workers = 4 + gateway_connection_pool_enabled = true + gateway_pool_max_db_connections = 8 + gateway_pool_max_client_conn = 250 + } + + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + local.gateway_pool_env[0].name == "LITELLM_PGBOUNCER_ENABLED" && local.gateway_pool_env[0].value == "true", + local.gateway_pool_env[1].name == "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS" && local.gateway_pool_env[1].value == "8", + local.gateway_pool_env[2].name == "LITELLM_PGBOUNCER_MAX_CLIENT_CONN" && local.gateway_pool_env[2].value == "250", + ]) + error_message = "The pool env must carry the enabled flag and the configured sizes as strings." + } + + assert { + condition = alltrue([ + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name], "LITELLM_PGBOUNCER_ENABLED"), + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name], "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS"), + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name], "LITELLM_PGBOUNCER_MAX_CLIENT_CONN"), + ]) + error_message = "The gateway service must receive all three LITELLM_PGBOUNCER_* env vars." + } + + assert { + condition = !anytrue(concat( + [for e in google_cloud_run_v2_service.backend[0].template[0].containers[0].env : startswith(e.name, "LITELLM_PGBOUNCER_")], + [for e in google_cloud_run_v2_job.migrations[0].template[0].template[0].containers[0].env : startswith(e.name, "LITELLM_PGBOUNCER_")], + )) + error_message = "The backend service and the migrations job must keep their direct database connection." + } +} + +run "pool_enabled_uses_the_module_default_sizes" { + command = plan + + variables { + gateway_connection_pool_enabled = true + } + + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + local.gateway_pool_env[1].name == "LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS" && local.gateway_pool_env[1].value == "20", + local.gateway_pool_env[2].name == "LITELLM_PGBOUNCER_MAX_CLIENT_CONN" && local.gateway_pool_env[2].value == "1000", + ]) + error_message = "The pool env must fall back to the module defaults of 20 upstream and 1000 client connections." + } +} + +run "gateway_starts_through_the_pool_aware_launcher" { + command = plan + + variables { + gateway_num_workers = 4 + } + + assert { + condition = alltrue([ + strcontains(local.gateway_launch_cmd, "exec python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), + strcontains(local.gateway_launch_cmd, "exec ddtrace-run python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), + !strcontains(local.gateway_launch_cmd, "uvicorn gateway.main:app"), + endswith(google_cloud_run_v2_service.gateway[0].template[0].containers[0].args[0], local.gateway_launch_cmd), + ]) + error_message = "The gateway must start through gateway.launch (with and without ddtrace) so the pooler starts once before uvicorn forks the workers." + } + + assert { + condition = strcontains(local.backend_launch_cmd, "uvicorn backend.main:app") + error_message = "The backend has no workers to share a pooler and keeps starting uvicorn directly." + } +} + +run "pool_sizes_below_one_fail_at_plan" { + command = plan + + variables { + gateway_connection_pool_enabled = true + gateway_pool_max_db_connections = 0 + gateway_pool_max_client_conn = 0 + } + + expect_failures = [ + var.gateway_pool_max_db_connections, + var.gateway_pool_max_client_conn, + ] +} diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index a753b4584a6..f298a0431c0 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -206,6 +206,45 @@ variable "gateway_num_workers" { } } +variable "gateway_connection_pool_enabled" { + description = <<-EOT + Run an in-container PgBouncer (transaction mode, loopback) in each gateway + instance, shared by every uvicorn worker. Without it each of the + `gateway_num_workers` workers opens its own Prisma pool straight to + Cloud SQL, so an instance's footprint against the database connection + ceiling is workers x connection_limit and grows with every instance. Sets + LITELLM_PGBOUNCER_ENABLED / LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / + LITELLM_PGBOUNCER_MAX_CLIENT_CONN on the gateway container only. The + module's Cloud SQL authenticates with the static password in Secret + Manager, which is what the pooler needs. Mirrors the AWS stack's + gateway_connection_pool_enabled. + EOT + type = bool + default = false +} + +variable "gateway_pool_max_db_connections" { + description = "Upstream Cloud SQL connections one gateway instance may hold when gateway_connection_pool_enabled is set, regardless of gateway_num_workers. 20 suits 4 workers." + type = number + default = 20 + + validation { + condition = var.gateway_pool_max_db_connections >= 1 + error_message = "gateway_pool_max_db_connections must be >= 1." + } +} + +variable "gateway_pool_max_client_conn" { + description = "Client connections the in-container PgBouncer accepts from the gateway workers when gateway_connection_pool_enabled is set." + type = number + default = 1000 + + validation { + condition = var.gateway_pool_max_client_conn >= 1 + error_message = "gateway_pool_max_client_conn must be >= 1." + } +} + # Cloud Run autoscales out of the box (request-rate driven). The min/max # bounds mirror the HPA replica bounds in helm/litellm/values.yaml so each # stack scales over the same range. Cloud Run has no direct CPU-utilization diff --git a/tests/test_gateway/test_launch.py b/tests/test_gateway/test_launch.py new file mode 100644 index 00000000000..964a7021416 --- /dev/null +++ b/tests/test_gateway/test_launch.py @@ -0,0 +1,155 @@ +import os +import socket +import sys +import textwrap +import urllib.parse +from pathlib import Path +from typing import Final, cast + +import pytest +from uvicorn.importer import import_from_string +from uvicorn.main import main as uvicorn_main + +import gateway.main +from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from litellm.proxy.db.db_url_settings import DatabaseURLSettings +from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings + +DB_ENV: Final = { + "DATABASE_HOST": "db.internal", + "DATABASE_PORT": "5432", + "DATABASE_USER": "litellm_pool", + "DATABASE_NAME": "litellm", + "DATABASE_PASSWORD": "p@ss", +} + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return cast(tuple[str, int], probe.getsockname())[1] + + +def _fake_pooler(tmp_path: Path) -> Path: + script: Final = tmp_path / "fake-pgbouncer" + script.write_text( + textwrap.dedent( + f"""\ + #!{sys.executable} + import configparser, select, socket, sys + if sys.argv[1:] == ["--version"]: + print("PgBouncer 1.25.2") + sys.exit(0) + ini = configparser.ConfigParser() + ini.read(sys.argv[1]) + port = ini.getint("pgbouncer", "listen_port") + tcp = socket.socket() + tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + tcp.bind(("127.0.0.1", port)) + tcp.listen() + unix = socket.socket(socket.AF_UNIX) + unix.bind(ini.get("pgbouncer", "unix_socket_dir") + f"/.s.PGSQL.{{port}}") + unix.listen() + while True: + for ready in select.select([tcp, unix], [], [])[0]: + ready.accept()[0].close() + """ + ) + ) + script.chmod(0o700) + return script + + +def _query(url: str) -> dict[str, str]: + return dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)) + + +@pytest.fixture +def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: + for var in ("DATABASE_URL", "IAM_TOKEN_DB_AUTH", "AZURE_POSTGRESQL_AUTH", "DATABASE_HOST_READ_REPLICA"): + monkeypatch.setenv(var, "") + monkeypatch.delenv(var) + for var, value in DB_ENV.items(): + monkeypatch.setenv(var, value) + return dict(DB_ENV) + + +def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: + return uvicorn_main.make_context("uvicorn", list(argv)).params + + +class TestUvicornArgv: + def test_keepalive_env_reaches_uvicorn(self): + params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"})) + assert params["app"] == GATEWAY_APP + assert params["workers"] == 4 + assert params["timeout_keep_alive"] == 75 + + def test_unset_env_keeps_the_uvicorn_default(self): + assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5 + + def test_an_explicit_flag_wins_over_the_env(self): + argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}) + assert _uvicorn_params(argv)["timeout_keep_alive"] == 30 + + def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self): + assert import_from_string(cast(str, _uvicorn_params(uvicorn_argv((), {}))["app"])) is gateway.main.app + + +class TestPoolDatabaseUrl: + def test_a_disabled_pooler_yields_no_url_to_install(self, password_env: dict[str, str]): + settings: Final = DatabaseURLSettings.from_env() + settings.apply_to_env() + environ: Final = {"DATABASE_URL": "postgresql://litellm_pool:p%40ss@db.internal:5432/litellm"} + assert pool_database_url(settings, PgBouncerSettings(enabled=False), environ) is None + + def test_a_missing_upstream_url_is_reported(self, password_env: dict[str, str]): + environ: Final[dict[str, str]] = {} + outcome: Final = pool_database_url(DatabaseURLSettings.from_env(), PgBouncerSettings(enabled=True), environ) + assert isinstance(outcome, PgBouncerError) + assert "DATABASE_URL" in outcome.reason + + def test_token_auth_is_refused(self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + environ: Final = {"DATABASE_URL": "postgresql://litellm:token@db.internal:5432/litellm"} + outcome: Final = pool_database_url( + DatabaseURLSettings.from_env(), + PgBouncerSettings(enabled=True, port=_free_port(), binary=str(_fake_pooler(tmp_path))), + environ, + ) + assert isinstance(outcome, PgBouncerError) + assert "IAM_TOKEN_DB_AUTH" in outcome.reason + + +class TestMain: + def test_workers_inherit_the_loopback_url_the_supervisor_installed( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + port: Final = _free_port() + monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") + monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port)) + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path))) + monkeypatch.setenv("KEEPALIVE_TIMEOUT", "75") + served: Final[list[tuple[str, ...]]] = [] + main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) + + pooled: Final = os.environ["DATABASE_URL"] + assert urllib.parse.urlsplit(pooled).netloc == f"litellm_pool:p%40ss@127.0.0.1:{port}" + assert _query(pooled)["pgbouncer"] == "true" + assert _uvicorn_params(served[0])["timeout_keep_alive"] == 75 + + DatabaseURLSettings.from_env().apply_to_env() + worker_url: Final = os.environ["DATABASE_URL"] + assert urllib.parse.urlsplit(worker_url).netloc == f"litellm_pool:p%40ss@127.0.0.1:{port}" + assert _query(worker_url)["pgbouncer"] == "true" + + def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(tmp_path / "missing-pgbouncer")) + served: Final[list[tuple[str, ...]]] = [] + with pytest.raises(SystemExit) as stopped: + main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) + assert "missing-pgbouncer" in str(stopped.value) + assert served == [] diff --git a/tests/test_litellm/proxy/db/test_pgbouncer.py b/tests/test_litellm/proxy/db/test_pgbouncer.py new file mode 100644 index 00000000000..da1d4c91f09 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_pgbouncer.py @@ -0,0 +1,624 @@ +import configparser +import logging +import os +import signal +import socket +import stat +import sys +import tempfile +import textwrap +import time +import urllib.parse +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final, cast + +import pytest + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.pgbouncer import ( + PgBouncerError, + PgBouncerPlan, + PgBouncerProcess, + PgBouncerSettings, + pgbouncer_version, + plan_pgbouncer, + start_in_container_pgbouncer, + unix_socket_path, + write_pgbouncer_files, +) + +UPSTREAM: Final = ( + "postgresql://app:p%40ss%27w@db.internal:5433/litellm" + "?schema=public&connection_limit=10&pool_timeout=20" + "&sslmode=require&sslaccept=strict&sslcert=/certs/ca.pem" + "&options=-c%20statement_timeout%3D7000%20-c%20lock_timeout%3D3000" +) +SETTINGS: Final = PgBouncerSettings(enabled=True, port=6543, max_db_connections=8, max_client_conn=400) + + +def _plan(url: str = UPSTREAM, run_as_user: str | None = None) -> PgBouncerPlan: + plan: Final = plan_pgbouncer(url, SETTINGS, Path("/run/pgb"), run_as_user) + assert isinstance(plan, PgBouncerPlan), plan + return plan + + +def _ini(plan: PgBouncerPlan) -> configparser.ConfigParser: + parser: Final = configparser.ConfigParser(interpolation=None) + parser.read_string(plan.ini) + return parser + + +def _query(url: str) -> dict[str, str]: + return dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query, keep_blank_values=True)) + + +class TestPlanPgBouncer: + def test_upstream_credentials_and_timeouts_move_into_the_pgbouncer_config(self): + ini: Final = _ini(_plan()) + assert ini["databases"]["litellm"] == ( + "host='db.internal' port=5433 dbname='litellm' user='app' password='p@ss''w' " + "connect_query='SET statement_timeout TO ''7000''; SET lock_timeout TO ''3000'''" + ) + assert _plan().userlist == '"app" "p@ss\'w"\n' + + def test_an_upstream_without_a_port_is_reached_on_the_postgres_default(self): + ini: Final = _ini(_plan("postgresql://app:pw@db/litellm")) + assert ini["databases"]["litellm"] == "host='db' port=5432 dbname='litellm' user='app' password='pw'" + + def test_pool_is_sized_from_settings_in_transaction_mode(self): + pgb: Final = _ini(_plan())["pgbouncer"] + assert pgb["pool_mode"] == "transaction" + assert pgb["max_db_connections"] == "8" + assert pgb["default_pool_size"] == "8" + assert pgb["max_client_conn"] == "400" + assert pgb["auth_type"] == "scram-sha-256" + assert pgb["listen_addr"] == "127.0.0.1" + assert pgb["listen_port"] == "6543" + assert pgb["auth_file"] == "/run/pgb/userlist.txt" + assert pgb["unix_socket_dir"] == "/run/pgb" + + def test_the_app_user_can_read_the_pgbouncer_console(self): + assert _ini(_plan())["pgbouncer"]["stats_users"] == "app" + + @pytest.mark.parametrize("user", ["app,admin", "app%20admin", "app%09admin"]) + def test_a_user_pgbouncer_would_split_into_several_console_users_is_refused(self, user: str): + outcome: Final = plan_pgbouncer(f"postgresql://{user}:pw@db/litellm", SETTINGS, Path("/run/pgb"), None) + assert isinstance(outcome, PgBouncerError) + assert "stats_users" in outcome.reason + + def test_pooled_url_points_prisma_at_loopback_without_prepared_statements(self): + pooled: Final = urllib.parse.urlsplit(_plan().pooled_url) + assert (pooled.hostname, pooled.port, pooled.path) == ("127.0.0.1", 6543, "/litellm") + assert (pooled.username, pooled.password) == ("app", "p%40ss%27w") + assert _query(_plan().pooled_url) == { + "schema": "public", + "connection_limit": "10", + "pool_timeout": "20", + "pgbouncer": "true", + } + + @pytest.mark.parametrize("hop_param", ["channel_binding=require", "gssencmode=require"]) + def test_transport_params_for_the_postgres_hop_stay_off_the_plain_tcp_loopback_url(self, hop_param: str): + pooled: Final = _plan(f"postgresql://app:pw@db/litellm?connection_limit=5&{hop_param}").pooled_url + assert _query(pooled) == {"connection_limit": "5", "pgbouncer": "true"} + + def test_verified_tls_becomes_server_side_verify_full_with_a_ca_copy_in_the_runtime_dir(self): + plan: Final = _plan() + pgb: Final = _ini(plan)["pgbouncer"] + assert pgb["server_tls_sslmode"] == "verify-full" + assert pgb["server_tls_ca_file"] == "/run/pgb/server-ca.pem" + assert plan.ca_source == "/certs/ca.pem" + + def test_unverified_require_stays_require_without_a_ca_file(self): + plan: Final = _plan("postgresql://app:pw@db/litellm?sslmode=require") + pgb: Final = _ini(plan)["pgbouncer"] + assert pgb["server_tls_sslmode"] == "require" + assert "server_tls_ca_file" not in pgb + assert plan.ca_source is None + + def test_no_tls_params_default_to_prefer(self): + assert _ini(_plan("postgresql://app:pw@db/litellm"))["pgbouncer"]["server_tls_sslmode"] == "prefer" + + def test_verification_without_a_ca_bundle_is_refused(self): + outcome: Final = plan_pgbouncer( + "postgresql://app:pw@db/litellm?sslmode=require&sslaccept=strict", SETTINGS, Path("/run/pgb"), None + ) + assert isinstance(outcome, PgBouncerError) + assert "sslcert" in outcome.reason + + def test_client_certificates_are_refused(self): + outcome: Final = plan_pgbouncer( + "postgresql://app:pw@db/litellm?sslidentity=/certs/client.p12", SETTINGS, Path("/run/pgb"), None + ) + assert isinstance(outcome, PgBouncerError) + assert "sslidentity" in outcome.reason + + @pytest.mark.parametrize( + "url", + [ + "postgresql://app@db/litellm", + "postgresql://app:pw@db", + "postgresql://:pw@db/litellm", + ], + ) + def test_urls_missing_forwardable_credentials_are_refused(self, url: str): + outcome: Final = plan_pgbouncer(url, SETTINGS, Path("/run/pgb"), None) + assert isinstance(outcome, PgBouncerError) + + def test_every_options_spelling_becomes_a_set_statement(self): + options: Final = urllib.parse.quote("-c a=1 -cb=2 --c=3") + ini: Final = _ini(_plan(f"postgresql://app:pw@db/litellm?options={options}")) + assert ini["databases"]["litellm"].endswith("connect_query='SET a TO ''1''; SET b TO ''2''; SET c TO ''3'''") + + def test_options_that_are_not_settings_are_refused(self): + outcome: Final = plan_pgbouncer( + "postgresql://app:pw@db/litellm?options=-c%20search_path", SETTINGS, Path("/run/pgb"), None + ) + assert isinstance(outcome, PgBouncerError) + assert "options" in outcome.reason + + def test_run_as_user_is_only_written_when_given(self): + assert _ini(_plan(run_as_user="nobody"))["pgbouncer"]["user"] == "nobody" + assert "user" not in _ini(_plan())["pgbouncer"] + + +class TestWritePgBouncerFiles: + def test_files_hold_the_plan_and_are_private_to_the_owner(self, tmp_path: Path): + plan: Final = _plan("postgresql://app:pw@db/litellm") + ini_path: Final = write_pgbouncer_files(plan, tmp_path, None) + assert isinstance(ini_path, Path), ini_path + assert ini_path == tmp_path / "pgbouncer.ini" + assert ini_path.read_text() == plan.ini + assert (tmp_path / "userlist.txt").read_text() == plan.userlist + for path in (ini_path, tmp_path / "userlist.txt"): + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert not (tmp_path / "server-ca.pem").exists() + + def test_the_ca_bundle_is_copied_next_to_the_ini_pgbouncer_reads(self, tmp_path: Path): + bundle: Final = tmp_path / "rds-root.pem" + bundle.write_text("-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n") + runtime_dir: Final = tmp_path / "run" + runtime_dir.mkdir() + plan: Final = plan_pgbouncer( + f"postgresql://app:pw@db/litellm?sslmode=verify-full&sslcert={bundle}", SETTINGS, runtime_dir, None + ) + assert isinstance(plan, PgBouncerPlan), plan + ini_path: Final = write_pgbouncer_files(plan, runtime_dir, None) + assert isinstance(ini_path, Path), ini_path + ca_file: Final = Path(_ini(plan)["pgbouncer"]["server_tls_ca_file"]) + assert ca_file.parent == runtime_dir + assert ca_file.read_text() == bundle.read_text() + + def test_an_unreadable_ca_bundle_is_reported(self, tmp_path: Path): + plan: Final = plan_pgbouncer( + f"postgresql://app:pw@db/litellm?sslmode=verify-full&sslcert={tmp_path / 'missing.pem'}", + SETTINGS, + tmp_path, + None, + ) + assert isinstance(plan, PgBouncerPlan), plan + outcome: Final = write_pgbouncer_files(plan, tmp_path, None) + assert isinstance(outcome, PgBouncerError) + assert "missing.pem" in outcome.reason + assert not (tmp_path / "pgbouncer.ini").exists() + + +def _bound_port(sock: socket.socket) -> int: + return cast(tuple[str, int], sock.getsockname())[1] + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return _bound_port(probe) + + +def _fake_pooler( + tmp_path: Path, + port: int, + exit_immediately: bool = False, + port_file: Path | None = None, + bind_delay_seconds: float = 0.0, + version_banner: str = "PgBouncer 1.25.2\nlibevent 2.1.13-stable", +) -> Path: + """An executable that listens like PgBouncer: on the TCP port first, then on ``.s.PGSQL.`` in the socket dir. + + Port and socket dir come from the ini it is given, else from ``port`` and + ``tmp_path``. With ``port_file`` each start reads the port from that file + instead. ``bind_delay_seconds`` holds the bind back, like a slow start. + ``--version`` prints ``version_banner``. + """ + script: Final = tmp_path / "fake-pgbouncer" + script.write_text( + textwrap.dedent( + f"""\ + #!{sys.executable} + import configparser, os, pathlib, select, socket, sys, time + if sys.argv[1:] == ["--version"]: + print({version_banner!r}) + sys.exit(0) + if {exit_immediately!r}: + sys.exit(3) + ini = configparser.ConfigParser() + ini.read(sys.argv[1:2]) + port = ini.getint("pgbouncer", "listen_port", fallback={port}) + if not {port_file is None!r}: + port = int(pathlib.Path({str(port_file)!r}).read_text()) + socket_dir = ini.get("pgbouncer", "unix_socket_dir", fallback={str(tmp_path)!r}) + socket_path = f"{{socket_dir}}/.s.PGSQL.{{port}}" + time.sleep({bind_delay_seconds!r}) + listener = socket.socket() + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", port)) + listener.listen() + if os.path.exists(socket_path): + os.unlink(socket_path) + unix_listener = socket.socket(socket.AF_UNIX) + unix_listener.bind(socket_path) + unix_listener.listen() + while True: + for ready in select.select([listener, unix_listener], [], [])[0]: + conn, _ = ready.accept() + conn.close() + """ + ) + ) + script.chmod(0o700) + return script + + +def _listening(port: int) -> bool: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + return True + except OSError: + return False + + +def _wait_until(condition: Callable[[], bool], timeout_seconds: float = 5.0) -> bool: + deadline: Final = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if condition(): + return True + time.sleep(0.05) + return False + + +class TestPgBouncerProcess: + def test_start_waits_for_the_listener_and_stop_ends_it(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port) + ) + assert pooler.start() is None + assert _listening(port) + pid: Final = pooler.pid + assert pid is not None + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + def test_a_crashed_pooler_is_restarted_with_a_new_pid(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.1, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + + def test_a_failed_restart_is_retried_until_the_pooler_is_back( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ): + port: Final = _free_port() + script: Final = _fake_pooler(tmp_path, port) + pooler: Final = PgBouncerProcess( + argv=(str(script),), port=port, socket_path=unix_socket_path(tmp_path, port), restart_delay_seconds=0.1 + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + hidden: Final = script.rename(tmp_path / "hidden") + with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: any("could not be restarted" in record.message for record in caplog.records)) + assert not _listening(port) + hidden.rename(script) + assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + + def test_a_replacement_that_never_listens_is_replaced_again(self, tmp_path: Path, caplog: pytest.LogCaptureFixture): + port: Final = _free_port() + port_file: Final = tmp_path / "port" + port_file.write_text(str(port)) + script: Final = _fake_pooler(tmp_path, port, port_file=port_file) + pooler: Final = PgBouncerProcess( + argv=(str(script),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.1, + ready_timeout_seconds=0.3, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + wrong_port: Final = _free_port() + port_file.write_text(str(wrong_port)) + with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: _listening(wrong_port)) + assert _wait_until(lambda: any("did not start listening" in record.message for record in caplog.records)) + port_file.write_text(str(port)) + assert _wait_until(lambda: _listening(port)) + assert _wait_until(lambda: not _listening(wrong_port)) + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + + def test_stopping_during_the_restart_delay_leaves_no_pooler_behind(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.3, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: not _listening(port)) + pooler.stop() + time.sleep(1.0) + assert not _listening(port) + assert pooler.pid == first_pid + + def test_a_stopped_pooler_is_not_restarted(self, tmp_path: Path, caplog: pytest.LogCaptureFixture): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.1, + ) + assert pooler.start() is None + with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + pooler.stop() + time.sleep(0.5) + assert not _listening(port) + assert caplog.records == [] + + def test_a_pooler_that_exits_during_startup_is_reported(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port, exit_immediately=True)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + ) + outcome: Final = pooler.start() + assert isinstance(outcome, PgBouncerError) + assert "status 3" in outcome.reason + + def test_a_missing_binary_is_reported(self, tmp_path: Path): + outcome: Final = PgBouncerProcess( + argv=("/nonexistent/pgbouncer",), port=_free_port(), socket_path=tmp_path / "sock" + ).start() + assert isinstance(outcome, PgBouncerError) + assert "/nonexistent/pgbouncer" in outcome.reason + + def test_a_port_owned_by_someone_else_is_refused_before_spawning(self, tmp_path: Path): + with socket.socket() as squatter: + squatter.bind(("127.0.0.1", 0)) + squatter.listen() + port: Final = _bound_port(squatter) + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), port=port, socket_path=unix_socket_path(tmp_path, port) + ) + outcome: Final = pooler.start() + assert isinstance(outcome, PgBouncerError) + assert f"127.0.0.1:{port} is already in use" in outcome.reason + assert pooler.pid is None + + def test_a_replacement_waits_until_a_squatter_leaves_the_port( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.5, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: not _listening(port)) + with socket.socket() as squatter, caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + squatter.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + squatter.bind(("127.0.0.1", port)) + squatter.listen() + assert _wait_until(lambda: any("already in use" in record.message for record in caplog.records)) + assert pooler.pid == first_pid + assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) + pooler.stop() + assert _wait_until(lambda: not _listening(port)) + + def test_a_listener_that_grabs_the_port_after_the_spawn_is_not_taken_for_the_pooler(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=0.5)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + ready_timeout_seconds=3.0, + ) + with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter: + squatter.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + starting: Final = starter.submit(pooler.start) + assert _wait_until(lambda: pooler.pid is not None) + squatter.bind(("127.0.0.1", port)) + squatter.listen() + outcome: Final = starting.result() + assert isinstance(outcome, PgBouncerError) + assert "exited with status 1" in outcome.reason + + def test_a_port_served_by_a_stranger_while_the_pooler_is_still_starting_is_reported(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port, bind_delay_seconds=30.0)),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + ready_timeout_seconds=0.5, + ) + with socket.socket() as squatter, ThreadPoolExecutor(max_workers=1) as starter: + squatter.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + starting: Final = starter.submit(pooler.start) + assert _wait_until(lambda: pooler.pid is not None) + squatter.bind(("127.0.0.1", port)) + squatter.listen() + outcome: Final = starting.result() + assert isinstance(outcome, PgBouncerError) + assert f"127.0.0.1:{port} is served by another process" in outcome.reason + pid: Final = pooler.pid + assert pid is not None + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + def test_a_pooler_that_never_listens_times_out(self, tmp_path: Path): + port: Final = _free_port() + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, _free_port())),), + port=port, + socket_path=unix_socket_path(tmp_path, port), + ready_timeout_seconds=0.5, + ) + outcome: Final = pooler.start() + assert isinstance(outcome, PgBouncerError) + assert "did not start listening" in outcome.reason + pid: Final = pooler.pid + assert pid is not None + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + +def _runtime_dir_listening_on(port: int) -> Path: + matches: Final = tuple( + ini.parent + for ini in Path(tempfile.gettempdir()).glob("litellm-pgbouncer-*/pgbouncer.ini") + if f"listen_port = {port}\n" in ini.read_text() + ) + assert len(matches) == 1, matches + return matches[0] + + +class TestStartInContainerPgBouncer: + def test_returns_the_loopback_url_once_the_pooler_listens(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm?connection_limit=5") + assert pooled == f"postgresql://app:pw@127.0.0.1:{port}/litellm?connection_limit=5&pgbouncer=true" + assert _listening(port) + + @pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning") + def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + exit_hooks: Final[list[Callable[[], None]]] = [] + pooled: Final = start_in_container_pgbouncer( + settings, "postgresql://app:pw@db/litellm", register_exit_hook=exit_hooks.append + ) + assert isinstance(pooled, str), pooled + runtime_dir: Final = _runtime_dir_listening_on(port) + + worker: Final = os.fork() + if worker == 0: + try: + for hook in exit_hooks: + hook() + finally: + os._exit(0) + if not _wait_until(lambda: os.waitpid(worker, os.WNOHANG) != (0, 0)): + os.kill(worker, signal.SIGKILL) + pytest.fail("the forked worker did not exit: an exit hook blocked on state inherited from the parent") + assert _listening(port) + assert (runtime_dir / "pgbouncer.ini").exists() + + for hook in exit_hooks: + hook() + assert _wait_until(lambda: not _listening(port)) + assert not runtime_dir.exists() + + def test_a_bad_upstream_url_is_reported_without_starting_anything(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app@db/litellm") + assert isinstance(outcome, PgBouncerError) + assert not _listening(port) + + def test_token_auth_is_refused_without_starting_anything(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + outcome: Final = start_in_container_pgbouncer( + settings, "postgresql://app:pw@db/litellm", token_auth_enabled=True + ) + assert isinstance(outcome, PgBouncerError) + assert "IAM_TOKEN_DB_AUTH" in outcome.reason + assert "AZURE_POSTGRESQL_AUTH" in outcome.reason + assert not _listening(port) + + def test_a_pgbouncer_that_survives_a_failed_tcp_bind_is_refused_without_starting(self, tmp_path: Path): + port: Final = _free_port() + binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.18.1\nlibevent 2.1.12-stable") + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) + outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") + assert isinstance(outcome, PgBouncerError) + assert "PgBouncer 1.18" in outcome.reason + assert "1.19" in outcome.reason + assert not _listening(port) + + def test_the_first_version_that_dies_on_a_failed_tcp_bind_is_accepted(self, tmp_path: Path): + port: Final = _free_port() + binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.19.0") + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) + assert start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") == ( + f"postgresql://app:pw@127.0.0.1:{port}/litellm?pgbouncer=true" + ) + assert _listening(port) + + +class TestPgBouncerVersion: + def test_reads_major_and_minor_from_the_banner(self, tmp_path: Path): + assert pgbouncer_version(str(_fake_pooler(tmp_path, _free_port()))) == (1, 25) + + def test_a_binary_that_cannot_run_is_reported(self, tmp_path: Path): + outcome: Final = pgbouncer_version(str(tmp_path / "missing-pgbouncer")) + assert isinstance(outcome, PgBouncerError) + assert "missing-pgbouncer" in outcome.reason + + def test_a_banner_without_a_version_is_reported(self, tmp_path: Path): + outcome: Final = pgbouncer_version(str(_fake_pooler(tmp_path, _free_port(), version_banner="something else"))) + assert isinstance(outcome, PgBouncerError) + assert "something else" in outcome.reason + + +class TestPgBouncerSettings: + def test_reads_the_litellm_pgbouncer_env_vars(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") + monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", "7000") + monkeypatch.setenv("LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS", "12") + settings: Final = PgBouncerSettings() + assert (settings.enabled, settings.port, settings.max_db_connections) == (True, 7000, 12) + + def test_defaults_are_off(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) + assert PgBouncerSettings().enabled is False diff --git a/tests/test_litellm/test_component_entrypoint.py b/tests/test_litellm/test_component_entrypoint.py index 09837d2b233..0c2a533b8bc 100644 --- a/tests/test_litellm/test_component_entrypoint.py +++ b/tests/test_litellm/test_component_entrypoint.py @@ -38,12 +38,16 @@ _STUB_TEMPLATE = """#!/bin/sh _ENTRYPOINT_RE = re.compile(r"^ENTRYPOINT\s+(\[.*\])\s*$", re.MULTILINE) _CMD_RE = re.compile(r"^CMD\s+(\[.*\])\s*$", re.MULTILINE) _COPY_RE = re.compile(r"^COPY\s+(?!--from)(\S+)\s+(\S+)\s*$", re.MULTILINE) -_APP_TARGET_RE = re.compile(r"(?:gateway|backend)\.main:app") +_APP_TARGET_RE = re.compile(r"(?:gateway|backend)\.main:app|gateway\.launch") _TF_STRING_LOCAL_RE = re.compile(r'^\s*(\w+)\s*=\s*"((?:[^"\\]|\\.)*)"\s*$', re.MULTILINE) _TF_INTERPOLATION_RE = re.compile(r"\$\{(local|var)\.(\w+)\}") TERRAFORM_LAUNCH_SITES = {TERRAFORM_ECS: 2, TERRAFORM_CLOUDRUN: 2} TERRAFORM_VAR_STUBS = {"gateway_num_workers": "2"} +COMPONENT_LAUNCHERS = { + "gateway": ("python", "-m", "gateway.launch"), + "backend": ("uvicorn", "backend.main:app"), +} _MAX_INTERPOLATION_PASSES = 5 @@ -63,7 +67,7 @@ def _run_entrypoint( """Run `script` with stubbed executables on PATH and return the recorded lines.""" bin_dir = tmp_path / "bin" bin_dir.mkdir(parents=True) - _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "litellm")) + _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "python", "litellm")) record = tmp_path / "record.txt" env = { @@ -330,22 +334,63 @@ def test_entrypoint_script_has_no_carriage_returns() -> None: @pytest.mark.parametrize( - "dockerfile, app_target", + "dockerfile, launcher", [ - (GATEWAY_DOCKERFILE, "gateway.main:app"), - (BACKEND_DOCKERFILE, "backend.main:app"), + (GATEWAY_DOCKERFILE, "python -m gateway.launch"), + (BACKEND_DOCKERFILE, "uvicorn backend.main:app"), ], ) -def test_component_images_launch_uvicorn_through_the_entrypoint(dockerfile: Path, app_target: str) -> None: +def test_component_images_launch_uvicorn_through_the_entrypoint(dockerfile: Path, launcher: str) -> None: entrypoint = " ".join(_entrypoint_argv(dockerfile)) assert IMAGE_ENTRYPOINT_PATH in entrypoint, f"{dockerfile} bypasses the ddtrace-aware entrypoint" - assert app_target in entrypoint - assert entrypoint.index(IMAGE_ENTRYPOINT_PATH) < entrypoint.index("uvicorn"), ( + assert launcher in entrypoint + assert entrypoint.index(IMAGE_ENTRYPOINT_PATH) < entrypoint.index(launcher), ( f"{dockerfile} must invoke uvicorn through the entrypoint, not the other way around" ) +@pytest.mark.parametrize( + "use_ddtrace, num_workers, expected_exec, expected_args", + [ + (None, "4", "exec=python", "args=-m gateway.launch --workers 4 --host 0.0.0.0 --port 4000"), + (None, None, "exec=python", "args=-m gateway.launch --workers 1 --host 0.0.0.0 --port 4000"), + ("true", "4", "exec=ddtrace-run", "args=python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000"), + ], +) +def test_gateway_image_execs_the_supervisor_with_its_worker_count( + use_ddtrace: str | None, num_workers: str | None, expected_exec: str, expected_args: str, tmp_path: Path +) -> None: + """Run the gateway image's ENTRYPOINT + CMD and record what the container execs. + + The Dockerfile's `/app/...` script path is resolved to the checked-in script and `python` + is stubbed on PATH, so the assertion is on the argv `gateway.launch` receives, not on the + Dockerfile text. + """ + entrypoint = tuple( + part.replace(IMAGE_ENTRYPOINT_PATH, str(COMPONENT_ENTRYPOINT)) for part in _entrypoint_argv(GATEWAY_DOCKERFILE) + ) + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + _write_stubs(bin_dir, ("ddtrace-run", "python", "uvicorn")) + record = tmp_path / "record.txt" + overrides = {"USE_DDTRACE": use_ddtrace, "NUM_WORKERS": num_workers} + env = { + **{k: v for k, v in os.environ.items() if k not in ("DD_TRACE_OPENAI_ENABLED", *overrides)}, + **{k: v for k, v in overrides.items() if v is not None}, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "RECORD": str(record), + "PYTHONPATH": PYTHONPATH_SENTINEL, + } + + result = subprocess.run( + [*entrypoint, *_cmd_argv(GATEWAY_DOCKERFILE)], env=env, capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert tuple(record.read_text().splitlines())[:2] == (expected_exec, expected_args) + + @pytest.mark.parametrize("dockerfile", [GATEWAY_DOCKERFILE, BACKEND_DOCKERFILE]) def test_component_images_make_the_entrypoint_executable(dockerfile: Path) -> None: body = dockerfile.read_text() @@ -367,17 +412,18 @@ def test_terraform_launch_command_matches_the_script_contract( implementations under the same environment and asserts they agree on which binary is exec'd and on whether the openai integration is disabled. """ - app_target = f"{component}.main:app" + launcher = COMPONENT_LAUNCHERS[component] + app_target = " ".join(launcher[1:]) command = _resolve_tf_local(terraform_file, f"{component}_launch_cmd") bin_dir = tmp_path / "bin" bin_dir.mkdir(parents=True) - _write_stubs(bin_dir, ("ddtrace-run", "uvicorn")) + _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "python")) from_terraform = _run_shell_command(command, bin_dir, tmp_path / "terraform.txt", use_ddtrace) from_script = _run_entrypoint( COMPONENT_ENTRYPOINT, - ("uvicorn", app_target), + launcher, use_ddtrace=use_ddtrace, tmp_path=tmp_path / "script", ) @@ -387,12 +433,14 @@ def test_terraform_launch_command_matches_the_script_contract( ) assert from_terraform[2] == from_script[2], f"{terraform_file} disagrees with the script on the openai integration" assert app_target in from_terraform[1] + assert "gateway.main:app" not in from_terraform[1], f"{terraform_file} bypasses the gateway.launch supervisor" if use_ddtrace in TRUTHY_USE_DDTRACE: assert from_terraform[0] == "exec=ddtrace-run" + assert from_terraform[1].startswith(f"args={launcher[0]} ") assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=False" else: - assert from_terraform[0] == "exec=uvicorn" + assert from_terraform[0] == f"exec={launcher[0]}" assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=" From 7658cd53aaf5f8cc217b9a9a25891851eaa64cbc Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 10 Sep 2026 22:25:03 +0000 Subject: [PATCH 373/411] fix(caching): judge swallowed Redis failures per admission and throttle worker tracebacks per error type Swallowed failures are now reported to the breaker when the admitted call exits, so a call admitted before the breaker opened cannot refresh the open timer or knock out the recovery probe. The sync batch read raises the typed open-breaker error like its async twin so DualCache releases its batch reservations, and LoggingWorker throttles tracebacks per exception class instead of worker-wide Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/dual_cache.py | 3 + litellm/caching/redis_cache.py | 60 ++++++++++------- litellm/litellm_core_utils/logging_worker.py | 20 +++--- tests/test_litellm/caching/test_dual_cache.py | 20 ++++++ .../test_litellm/caching/test_redis_cache.py | 65 +++++++++++++++++-- .../litellm_core_utils/test_logging_worker.py | 32 ++++++++- 6 files changed, 162 insertions(+), 38 deletions(-) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 6fb918ce748..5dd0493cd65 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -206,6 +206,9 @@ class DualCache(BaseCache): redis_result: Final = self.redis_cache.batch_get_cache( key_list=sublist_keys, parent_otel_span=parent_otel_span ) + except RedisCircuitBreakerOpenError: + self._rollback_redis_batch_key_reservations(previous_access_times) + return result except Exception: # Do not throttle subsequent callers if the Redis read fails. self._rollback_redis_batch_key_reservations(previous_access_times) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 6c19b018c1d..3ecf805ed0e 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -285,7 +285,9 @@ class RedisCircuitBreaker: _RedisCallResult = TypeVar("_RedisCallResult") -_swallowed_redis_failures: Final[ContextVar[int]] = ContextVar("litellm_swallowed_redis_failures", default=0) +_swallowed_redis_failures: Final[ContextVar[tuple[bool, ...]]] = ContextVar( + "litellm_swallowed_redis_failures", default=() +) def _opaque_kwarg_key(value: object) -> str: @@ -405,8 +407,8 @@ def _breaker_metrics() -> _BreakerMetrics: return _BreakerMetrics() -def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseException) -> None: - """Record a Redis failure that the calling method is about to swallow. +def _record_swallowed_redis_failure(exc: BaseException) -> None: + """Note a Redis failure that the calling method is about to swallow, for the breaker exit to judge. The marker is a ContextVar rather than a counter on the breaker because breakers are shared by every concurrent caller. A plain shared counter cannot tell "my call failed" @@ -416,8 +418,7 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep """ if not _is_redis_health_failure(exc): return - breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc)) - _swallowed_redis_failures.set(_swallowed_redis_failures.get() + 1) + _swallowed_redis_failures.set((*_swallowed_redis_failures.get(), _is_redis_timeout_failure(exc))) @dataclass(frozen=True, slots=True) @@ -430,25 +431,42 @@ def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> _BreakerA """Reject the call if the breaker is open, else snapshot what its outcome will be judged against.""" if breaker.is_open(): raise RedisCircuitBreakerOpenError(f"Redis circuit breaker is open, skipping {name}") - return _BreakerAdmission(swallowed_before=_swallowed_redis_failures.get(), generation=breaker.generation) + return _BreakerAdmission(swallowed_before=len(_swallowed_redis_failures.get()), generation=breaker.generation) + + +def _take_swallowed_failures(admission: _BreakerAdmission) -> tuple[bool, ...]: + """Return the is_timeout flag of every failure this call swallowed, and drop them from the context.""" + all_swallowed: Final = _swallowed_redis_failures.get() + _swallowed_redis_failures.set(all_swallowed[: admission.swallowed_before]) + return all_swallowed[admission.swallowed_before :] def _exit_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmission) -> None: - """Record success only when nothing failed while the call ran and the breaker has not opened since. + """Report the call's outcome to the breaker generation it was admitted under. Several Redis methods catch their own connection errors and return a default, so a - method that returned is not on its own proof of a healthy Redis. + method that returned is not on its own proof of a healthy Redis. A call admitted + before the breaker opened reports nothing: its failures would refresh the open + timer or knock out the recovery probe, and its success would close it early. """ + swallowed: Final = _take_swallowed_failures(admission) if admission.generation != breaker.generation: return - if _swallowed_redis_failures.get() == admission.swallowed_before: + if not swallowed: breaker.record_success() + return + for is_timeout in swallowed: + breaker.record_failure(is_timeout=is_timeout) def _fail_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmission, exc: BaseException) -> None: - if admission.generation != breaker.generation or not _is_redis_health_failure(exc): + swallowed: Final = _take_swallowed_failures(admission) + if admission.generation != breaker.generation: return - breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc)) + for is_timeout in swallowed: + breaker.record_failure(is_timeout=is_timeout) + if _is_redis_health_failure(exc): + breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc)) async def _run_under_circuit_breaker( @@ -1056,7 +1074,7 @@ class RedisCache(BaseCache): str(e), value, ) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) async def _pipeline_helper( self, @@ -1143,7 +1161,7 @@ class RedisCache(BaseCache): str(e), cache_value, ) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) async def _set_cache_sadd_helper( self, @@ -1228,7 +1246,7 @@ class RedisCache(BaseCache): str(e), value, ) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) @_redis_circuit_breaker_guard async def batch_cache_write(self, key, value, **kwargs): @@ -1386,7 +1404,7 @@ class RedisCache(BaseCache): except Exception as e: # NON blocking - notify users Redis is throwing an exception verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: %s", e) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]: """ @@ -1423,9 +1441,9 @@ class RedisCache(BaseCache): key_value_dict = {} _key_list: Final = [key for key in key_list if key is not None] start_time: Final = time.time() + admission: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") try: - admission: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list] results: Final = self._run_redis_mget_operation(keys=_keys) _exit_circuit_breaker(self._circuit_breaker, admission) @@ -1452,8 +1470,6 @@ class RedisCache(BaseCache): decoded_results[k] = v return decoded_results - except RedisCircuitBreakerOpenError: - return key_value_dict except Exception as e: failed_at: Final = time.time() self.service_logger_obj.service_failure_hook( @@ -1466,7 +1482,7 @@ class RedisCache(BaseCache): parent_otel_span=parent_otel_span, ) verbose_logger.error("Error occurred in batch get cache - %s", e) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _fail_circuit_breaker(self._circuit_breaker, admission, e) return key_value_dict @_redis_circuit_breaker_guard @@ -1513,7 +1529,7 @@ class RedisCache(BaseCache): ) ) print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}") - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) @_redis_circuit_breaker_guard async def async_batch_get_cache( @@ -1585,7 +1601,7 @@ class RedisCache(BaseCache): ) ) verbose_logger.error("Error occurred in async batch get cache - %s", e) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) return key_value_dict def sync_ping(self) -> bool: @@ -1837,7 +1853,7 @@ class RedisCache(BaseCache): return ttl except Exception as e: verbose_logger.debug("Redis TTL Error: %s", e) - _record_swallowed_redis_failure(self._circuit_breaker, e) + _record_swallowed_redis_failure(e) return None @_redis_circuit_breaker_guard diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py index dfaace1bb10..de3999d63ea 100644 --- a/litellm/litellm_core_utils/logging_worker.py +++ b/litellm/litellm_core_utils/logging_worker.py @@ -55,8 +55,8 @@ class LoggingWorker: self.max_queue_size = max_queue_size self.concurrency = concurrency self.error_traceback_interval = error_traceback_interval - self._last_error_traceback_at: float | None = None - self._errors_since_traceback: int = 0 + self._last_error_traceback_at: dict[type[BaseException], float] = {} + self._errors_since_traceback: dict[type[BaseException], int] = {} self._queue: asyncio.Queue[LoggingTask] | None = None self._worker_task: asyncio.Task | None = None self._running_tasks: set[asyncio.Task] = set() @@ -178,19 +178,21 @@ class LoggingWorker: sem.release() def _log_task_error(self, error: Exception) -> None: - """One traceback per interval: a stalled backend fails every in-flight task at once.""" + """One traceback per error type per interval: a stalled backend fails every in-flight task at once.""" now: Final = time.monotonic() - last_traceback_at: Final = self._last_error_traceback_at + error_type: Final = type(error) + last_traceback_at: Final = self._last_error_traceback_at.get(error_type) if last_traceback_at is not None and now - last_traceback_at < self.error_traceback_interval: - self._errors_since_traceback += 1 + self._errors_since_traceback[error_type] = self._errors_since_traceback.get(error_type, 0) + 1 return verbose_logger.exception( - "LoggingWorker error (%d more suppressed since the last traceback): %r", - self._errors_since_traceback, + "LoggingWorker error (%d more %s suppressed since the last traceback): %r", + self._errors_since_traceback.get(error_type, 0), + error_type.__name__, error, ) - self._last_error_traceback_at = now - self._errors_since_traceback = 0 + self._last_error_traceback_at[error_type] = now + self._errors_since_traceback[error_type] = 0 async def _worker_loop(self) -> None: """Main worker loop that gets tasks and schedules them to run concurrently.""" diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 3e21362b3f0..b2462b297fc 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -653,3 +653,23 @@ def test_open_breaker_is_a_quiet_cache_miss_on_the_sync_read_path(dual_cache_wit noisy = [r for r in caplog.records if r.levelno > logging.DEBUG] assert noisy == [], f"an open breaker must be silent per call, got {[r.getMessage() for r in noisy]}" + + +def test_open_breaker_does_not_leave_sync_batch_reservations_behind(dual_cache_with_open_breaker, caplog): + """A batch read skipped by the breaker must not hold its keys for the batch expiry window. + + The sync read reserves keys before dialing Redis so concurrent callers do not all hit it. + When the breaker rejects the read, those reservations have to be released, otherwise the + first read after Redis recovers is still throttled for up to default_redis_batch_cache_expiry. + """ + caplog.set_level(logging.DEBUG, logger="LiteLLM") + dual_cache_with_open_breaker.redis_cache.redis_client.mget.side_effect = AssertionError( + "an open breaker must not touch Redis" + ) + + assert dual_cache_with_open_breaker.batch_get_cache(keys=["lit7468", "lit7460"]) == [None, None] + + assert "lit7468" not in dual_cache_with_open_breaker.last_redis_batch_access_time + assert "lit7460" not in dual_cache_with_open_breaker.last_redis_batch_access_time + noisy = [r for r in caplog.records if r.levelno > logging.DEBUG] + assert noisy == [], f"an open breaker must be silent per call, got {[r.getMessage() for r in noisy]}" diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 1a884d44db1..3bc3aa1e047 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -516,14 +516,20 @@ async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_met await call_method(cache) -def test_circuit_breaker_open_keeps_sync_batch_get_cache_as_a_miss(sync_batch_redis_cache): - """An open breaker must preserve the sync batch read's dictionary fallback.""" +def test_sync_batch_get_cache_swallowed_failures_open_the_breaker_and_then_fast_fail(sync_batch_redis_cache): + """The sync batch read hides its Redis error behind an empty dict, but the breaker must still + count it, and once open the read has to raise the typed error like its async twin so DualCache + can release the batch reservations it took before the call. + """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {} - assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {} + with pytest.raises(RedisCircuitBreakerOpenError): + sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) + assert sync_batch_redis_cache.redis_client.mget.call_count == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD def test_batch_get_counts_raises_where_batch_get_cache_reports_a_miss(sync_batch_redis_cache): @@ -647,6 +653,7 @@ def test_sync_batch_get_cache_survives_a_service_callback_that_raises( from concurrent.futures import ThreadPoolExecutor import litellm + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD cache, service_logger = sync_batch_cache_with_service_logger @@ -661,7 +668,8 @@ def test_sync_batch_get_cache_survives_a_service_callback_that_raises( with ThreadPoolExecutor(max_workers=1) as pool: assert pool.submit(cache.batch_get_cache, key_list=["lit6729"]).result() == {} - assert cache.batch_get_cache(key_list=["lit6729"]) == {} + with pytest.raises(RedisCircuitBreakerOpenError): + cache.batch_get_cache(key_list=["lit6729"]) def test_call_stack_info_skips_breaker_guard_frames(): @@ -801,7 +809,7 @@ async def test_concurrent_success_is_not_cancelled_by_another_calls_failure(): # starts would leave its snapshot correct and prove nothing. async def swallows_a_failure(): await asyncio.sleep(0.02) - _record_swallowed_redis_failure(breaker, RedisConnectionError("redis unreachable")) + _record_swallowed_redis_failure(RedisConnectionError("redis unreachable")) async def succeeds_while_the_other_fails(): await asyncio.sleep(0.05) @@ -1097,6 +1105,53 @@ async def test_success_admitted_before_the_breaker_opened_cannot_close_it(): await _run_under_circuit_breaker(breaker, "op", slow_success) +@pytest.mark.asyncio +async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay_recovery(): + """A stale in-flight call that swallows its Redis error must not restart the open timer. + + Guarded methods that return a default instead of raising still report their failure to + the breaker on exit. If that report landed against a newer breaker generation, every + slow call that was already dialing Redis when the breaker opened would push recovery + back by its own socket timeout, and knock out the HALF_OPEN probe if it landed then. + """ + from redis.exceptions import ConnectionError as RedisConnectionError + + from litellm.caching.redis_cache import ( + RedisCircuitBreaker, + _record_swallowed_redis_failure, + _run_under_circuit_breaker, + ) + + breaker = RedisCircuitBreaker(failure_threshold=2, recovery_timeout=0.05) + release_stale_call = asyncio.Event() + + async def stale_call_that_swallows_its_failure(): + await release_stale_call.wait() + _record_swallowed_redis_failure(RedisConnectionError("refused")) + return {} + + async def refused(): + raise RedisConnectionError("refused") + + async def recovered(): + return "ok" + + in_flight = asyncio.create_task(_run_under_circuit_breaker(breaker, "stale", stale_call_that_swallows_its_failure)) + await asyncio.sleep(0) + for _ in range(breaker.failure_threshold): + with pytest.raises(RedisConnectionError): + await _run_under_circuit_breaker(breaker, "op", refused) + assert breaker.is_open() is True + + await asyncio.sleep(0.04) + release_stale_call.set() + assert await in_flight == {} + await asyncio.sleep(0.03) + + assert await _run_under_circuit_breaker(breaker, "probe", recovered) == "ok" + assert breaker.is_open() is False + + @pytest.mark.asyncio async def test_open_breaker_raises_its_own_exception_type(): """Callers with an optional cache need to tell the expected fast-fail apart from a real error.""" diff --git a/tests/test_litellm/litellm_core_utils/test_logging_worker.py b/tests/test_litellm/litellm_core_utils/test_logging_worker.py index f93a2e0b08a..66d49b22c6b 100644 --- a/tests/test_litellm/litellm_core_utils/test_logging_worker.py +++ b/tests/test_litellm/litellm_core_utils/test_logging_worker.py @@ -568,5 +568,33 @@ class TestLoggingWorker: messages = [r.getMessage() for r in caplog.records if "LoggingWorker error" in r.getMessage()] assert len(messages) == 2, messages - assert "(0 more suppressed" in messages[0] - assert "(4 more suppressed" in messages[1] + assert "(0 more TimeoutError suppressed" in messages[0] + assert "(4 more TimeoutError suppressed" in messages[1] + + @pytest.mark.asyncio + async def test_traceback_throttle_is_per_error_type(self, caplog): + """A timeout burst from one stalled backend must not hide the first traceback of a different + failure, otherwise a misconfigured callback stays invisible for the whole interval. + """ + caplog.set_level(logging.DEBUG, logger="LiteLLM") + worker = LoggingWorker(timeout=0.05, max_queue_size=200, concurrency=100, error_traceback_interval=60.0) + worker.start() + + async def stalled_callback(): + await asyncio.sleep(10) + + async def misconfigured_callback(): + raise KeyError("missing api key") + + for _ in range(20): + worker.enqueue(stalled_callback()) + await asyncio.sleep(0.2) + for _ in range(3): + worker.enqueue(misconfigured_callback()) + await asyncio.sleep(0.2) + await worker.stop() + + messages = [r.getMessage() for r in caplog.records if "LoggingWorker error" in r.getMessage()] + assert len(messages) == 2, messages + assert "TimeoutError" in messages[0] + assert "KeyError" in messages[1] and "missing api key" in messages[1] From 368de5df688ef6d21ed47302905aa30e99ee84f5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:30:16 -0700 Subject: [PATCH 374/411] fix(proxy): log prompt-shaped model strings on pass-through failures under the placeholder model --- litellm/constants.py | 1 + .../spend_tracking/spend_tracking_utils.py | 13 +++++-- .../test_spend_tracking_utils.py | 37 ++++++++++++++----- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 33a46616c4e..bc3a82cb655 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2000,6 +2000,7 @@ NON_INFERENCE_CALL_TYPES: Final[frozenset[str]] = frozenset( ) UNKNOWN_MODEL_SPEND_LOG_MODEL: Final[str] = "unknown-model" +MAX_SPEND_LOG_MODEL_NAME_LENGTH: Final[int] = 256 # PTU reservation rollup writes rows to LiteLLM_DailyTeamSpend with this # sentinel api_key so PTU flat cost stays distinguishable from real per-request diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index ebe5b530e60..d2f44be6004 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -16,6 +16,7 @@ from litellm.constants import ( LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL, @@ -333,6 +334,10 @@ def _sl_attribution_fallback( return standard_logging_payload.get(field) or "" +def _looks_like_model_name(model: str) -> bool: + return len(model) <= MAX_SPEND_LOG_MODEL_NAME_LENGTH and not any(char.isspace() for char in model) + + def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogsPayload: if kwargs is None: kwargs = {} @@ -435,11 +440,13 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs or None ) raw_model: Final = cast(str, kwargs.get("model") or "") + resolved_model: Final = ( + standard_logging_payload.get("model") if standard_logging_payload is not None else None + ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) model_name: Final = ( UNKNOWN_MODEL_SPEND_LOG_MODEL - if rejected_as_unknown_model - else (standard_logging_payload.get("model") if standard_logging_payload is not None else None) - or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) + if rejected_as_unknown_model or not _looks_like_model_name(resolved_model) + else resolved_model ) litellm_call_id: Final = cast( str | None, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 7a20b9ab026..c225b35992f 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1,8 +1,8 @@ import asyncio import datetime import json -from datetime import timezone from collections.abc import Mapping +from datetime import timezone from typing import Any, Final, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -15,12 +15,13 @@ from litellm.constants import ( LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_tracking_utils import ( @@ -41,7 +42,6 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_logging_payload, get_spend_logs_id, ) -from litellm.proxy._types import SpendLogsPayload from litellm.proxy.utils import hash_token from litellm.types.utils import ( StandardLoggingHiddenParams, @@ -927,21 +927,40 @@ def test_safe_dumps_complex_metadata_like_object(): _RAW_MODEL_WITH_PROMPT: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" +_BEDROCK_INFERENCE_PROFILE_ARN: Final = ( + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-sonnet-4-5" +) +_OVERLONG_MODEL: Final = "m" * (MAX_SPEND_LOG_MODEL_NAME_LENGTH + 1) + + @pytest.mark.parametrize( - ("rejection", "expected_model"), + ("requested_model", "failure", "expected_model"), [ ( + _RAW_MODEL_WITH_PROMPT, ProxyModelNotFoundError(route="acompletion", model_name=_RAW_MODEL_WITH_PROMPT), UNKNOWN_MODEL_SPEND_LOG_MODEL, ), - (ValueError("provider timed out"), _RAW_MODEL_WITH_PROMPT), + ( + _RAW_MODEL_WITH_PROMPT, + ValueError("Upstream passthrough request failed with status 404"), + UNKNOWN_MODEL_SPEND_LOG_MODEL, + ), + (_OVERLONG_MODEL, ValueError("provider timed out"), UNKNOWN_MODEL_SPEND_LOG_MODEL), + ( + "gpt-5.2", + ProxyModelNotFoundError(route="acompletion", model_name="gpt-5.2"), + UNKNOWN_MODEL_SPEND_LOG_MODEL, + ), + ("gpt-5.2", ValueError("provider timed out"), "gpt-5.2"), + (_BEDROCK_INFERENCE_PROFILE_ARN, ValueError("provider timed out"), _BEDROCK_INFERENCE_PROFILE_ARN), ], ) -def test_get_logging_payload_replaces_model_only_when_router_rejected_it_as_unknown( - rejection: Exception, expected_model: str +def test_get_logging_payload_replaces_rejected_or_prompt_shaped_models_with_the_placeholder( + requested_model: str, failure: Exception, expected_model: str ): kwargs: Final = { - "model": _RAW_MODEL_WITH_PROMPT, + "model": requested_model, "messages": [{"role": "user", "content": "hi"}], "call_type": "acompletion", "litellm_params": {"metadata": {"user_api_key": "sk-test", "status": "failure"}}, @@ -949,7 +968,7 @@ def test_get_logging_payload_replaces_model_only_when_router_rejected_it_as_unkn payload: Final = get_logging_payload( kwargs=kwargs, - response_obj=rejection, + response_obj=failure, start_time=datetime.datetime.now(timezone.utc), end_time=datetime.datetime.now(timezone.utc), ) From 098bfe0bbc842f43caa15b6b2fc14e2a6106c301 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:36:32 -0700 Subject: [PATCH 375/411] fix(proxy): only remap prompt-shaped model strings on failed spend rows --- .../spend_tracking/spend_tracking_utils.py | 5 ++- .../test_spend_tracking_utils.py | 32 +++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index d2f44be6004..5690e43ab2b 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -443,9 +443,12 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs resolved_model: Final = ( standard_logging_payload.get("model") if standard_logging_payload is not None else None ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) + failed_with_prompt_shaped_model: Final = _get_status_for_spend_log( + metadata=metadata + ) == "failure" and not _looks_like_model_name(resolved_model) model_name: Final = ( UNKNOWN_MODEL_SPEND_LOG_MODEL - if rejected_as_unknown_model or not _looks_like_model_name(resolved_model) + if rejected_as_unknown_model or failed_with_prompt_shaped_model else resolved_model ) litellm_call_id: Final = cast( diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index c225b35992f..11cf7ac530e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -141,9 +141,7 @@ def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id(): from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1") - cost_session: Final = _get_batch_trace_session_id( - call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost" - ) + cost_session: Final = _get_batch_trace_session_id(call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost") assert create_session == cost_session == "batch-uid-1" @@ -976,6 +974,24 @@ def test_get_logging_payload_replaces_rejected_or_prompt_shaped_models_with_the_ assert payload["model"] == expected_model +def test_get_logging_payload_keeps_a_whitespace_model_name_on_success(): + kwargs: Final = { + "model": _RAW_MODEL_WITH_PROMPT, + "messages": [{"role": "user", "content": "hi"}], + "call_type": "acompletion", + "litellm_params": {"metadata": {"user_api_key": "sk-test"}}, + } + + payload: Final = get_logging_payload( + kwargs=kwargs, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["model"] == _RAW_MODEL_WITH_PROMPT + + @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_none(): @@ -4306,7 +4322,7 @@ ANTHROPIC_MESSAGES_SSE_CHUNKS: Final = ( 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},' '"usage":{"output_tokens":4}}\n\n', - "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + 'event: message_stop\ndata: {"type":"message_stop"}\n\n', ) @@ -4344,9 +4360,7 @@ def test_spend_log_request_id_is_the_message_id_a_non_streaming_messages_caller_ """ logging_obj = _anthropic_messages_logging_obj(stream=False) - logged_response = logging_obj._handle_anthropic_messages_response_logging( - result=ANTHROPIC_MESSAGES_RESPONSE - ) + logged_response = logging_obj._handle_anthropic_messages_response_logging(result=ANTHROPIC_MESSAGES_RESPONSE) assert logged_response.id == "msg_01Lit6806NonStreaming" assert ( @@ -4422,9 +4436,7 @@ def test_spend_log_request_id_still_falls_back_to_litellm_call_id_without_a_prov end_time=datetime.datetime.now(timezone.utc), logging_obj=logging_obj, ) - assert logging_obj.model_call_details["complete_streaming_response"].id == ( - "6806cafe-0000-4000-8000-000000000001" - ) + assert logging_obj.model_call_details["complete_streaming_response"].id == ("6806cafe-0000-4000-8000-000000000001") def test_spend_log_request_id_for_chat_completions_is_untouched(): From a93753e73bfc67de48a8b068d590b72b6d7425c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:43:27 -0700 Subject: [PATCH 376/411] fix(proxy): keep the model on failed rows that resolved to a configured model group --- .../spend_tracking/spend_tracking_utils.py | 8 +++-- .../test_spend_tracking_utils.py | 32 +++++++++++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 5690e43ab2b..f8cd73938b4 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -443,9 +443,11 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs resolved_model: Final = ( standard_logging_payload.get("model") if standard_logging_payload is not None else None ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) - failed_with_prompt_shaped_model: Final = _get_status_for_spend_log( - metadata=metadata - ) == "failure" and not _looks_like_model_name(resolved_model) + failed_with_prompt_shaped_model: Final = ( + _get_status_for_spend_log(metadata=metadata) == "failure" + and not _model_group + and not _looks_like_model_name(resolved_model) + ) model_name: Final = ( UNKNOWN_MODEL_SPEND_LOG_MODEL if rejected_as_unknown_model or failed_with_prompt_shaped_model diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 11cf7ac530e..03c34e9e198 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -141,7 +141,9 @@ def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id(): from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1") - cost_session: Final = _get_batch_trace_session_id(call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost") + cost_session: Final = _get_batch_trace_session_id( + call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost" + ) assert create_session == cost_session == "batch-uid-1" @@ -974,17 +976,29 @@ def test_get_logging_payload_replaces_rejected_or_prompt_shaped_models_with_the_ assert payload["model"] == expected_model -def test_get_logging_payload_keeps_a_whitespace_model_name_on_success(): +@pytest.mark.parametrize( + ("metadata", "response_obj"), + [ + ({"user_api_key": "sk-test"}, litellm.ModelResponse(id="chatcmpl-test", choices=[])), + ( + {"user_api_key": "sk-test", "model_group": "team alias", "status": "failure"}, + ValueError("provider timed out"), + ), + ], +) +def test_get_logging_payload_keeps_a_whitespace_model_name_on_success_or_a_routed_failure( + metadata: dict[str, str], response_obj: litellm.ModelResponse | Exception +): kwargs: Final = { "model": _RAW_MODEL_WITH_PROMPT, "messages": [{"role": "user", "content": "hi"}], "call_type": "acompletion", - "litellm_params": {"metadata": {"user_api_key": "sk-test"}}, + "litellm_params": {"metadata": metadata}, } payload: Final = get_logging_payload( kwargs=kwargs, - response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + response_obj=response_obj, start_time=datetime.datetime.now(timezone.utc), end_time=datetime.datetime.now(timezone.utc), ) @@ -4322,7 +4336,7 @@ ANTHROPIC_MESSAGES_SSE_CHUNKS: Final = ( 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},' '"usage":{"output_tokens":4}}\n\n', - 'event: message_stop\ndata: {"type":"message_stop"}\n\n', + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", ) @@ -4360,7 +4374,9 @@ def test_spend_log_request_id_is_the_message_id_a_non_streaming_messages_caller_ """ logging_obj = _anthropic_messages_logging_obj(stream=False) - logged_response = logging_obj._handle_anthropic_messages_response_logging(result=ANTHROPIC_MESSAGES_RESPONSE) + logged_response = logging_obj._handle_anthropic_messages_response_logging( + result=ANTHROPIC_MESSAGES_RESPONSE + ) assert logged_response.id == "msg_01Lit6806NonStreaming" assert ( @@ -4436,7 +4452,9 @@ def test_spend_log_request_id_still_falls_back_to_litellm_call_id_without_a_prov end_time=datetime.datetime.now(timezone.utc), logging_obj=logging_obj, ) - assert logging_obj.model_call_details["complete_streaming_response"].id == ("6806cafe-0000-4000-8000-000000000001") + assert logging_obj.model_call_details["complete_streaming_response"].id == ( + "6806cafe-0000-4000-8000-000000000001" + ) def test_spend_log_request_id_for_chat_completions_is_untouched(): From 032a9ebc4b370eb6f53ca00ce0755b069ddc1331 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 15:46:01 -0700 Subject: [PATCH 377/411] chore(github): disable blank issues so filers must use a template (#40629) --- .github/ISSUE_TEMPLATE/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index cbf380bac01..fc86c229c81 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,4 +1,4 @@ -blank_issues_enabled: true +blank_issues_enabled: false contact_links: - name: Schedule Demo url: https://enterprise.litellm.ai/demo From fbd923190e04583e56184c6ff38e1b2e41ad6c36 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 10 Sep 2026 22:53:46 +0000 Subject: [PATCH 378/411] fix(caching): let the outer breaker guard judge failures a nested guarded call swallowed batch_cache_write flushes through async_set_cache_pipeline, both guarded. The inner guard consumed the swallowed pipeline failure and the outer then recorded a success, so a dead Redis never tripped the breaker on the batch write path. Only the outermost admission now reports, and the sync batch read runs under the same guard. Also covers the sync add_cache and embedding pipeline wrappers, the increment pipeline, sadd, and the stale raise path in tests. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_cache.py | 36 ++++++-- tests/test_litellm/caching/test_caching.py | 33 +++++-- tests/test_litellm/caching/test_dual_cache.py | 5 ++ .../test_litellm/caching/test_redis_cache.py | 90 +++++++++++++++++-- 4 files changed, 141 insertions(+), 23 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 3ecf805ed0e..2c1ca75212c 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -288,6 +288,7 @@ _RedisCallResult = TypeVar("_RedisCallResult") _swallowed_redis_failures: Final[ContextVar[tuple[bool, ...]]] = ContextVar( "litellm_swallowed_redis_failures", default=() ) +_breaker_depth: Final[ContextVar[int]] = ContextVar("litellm_redis_breaker_depth", default=0) def _opaque_kwarg_key(value: object) -> str: @@ -425,13 +426,20 @@ def _record_swallowed_redis_failure(exc: BaseException) -> None: class _BreakerAdmission: swallowed_before: int generation: int + nested: bool def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> _BreakerAdmission: """Reject the call if the breaker is open, else snapshot what its outcome will be judged against.""" if breaker.is_open(): raise RedisCircuitBreakerOpenError(f"Redis circuit breaker is open, skipping {name}") - return _BreakerAdmission(swallowed_before=len(_swallowed_redis_failures.get()), generation=breaker.generation) + depth: Final = _breaker_depth.get() + _breaker_depth.set(depth + 1) + return _BreakerAdmission( + swallowed_before=len(_swallowed_redis_failures.get()), + generation=breaker.generation, + nested=depth > 0, + ) def _take_swallowed_failures(admission: _BreakerAdmission) -> tuple[bool, ...]: @@ -448,7 +456,14 @@ def _exit_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmis method that returned is not on its own proof of a healthy Redis. A call admitted before the breaker opened reports nothing: its failures would refresh the open timer or knock out the recovery probe, and its success would close it early. + + A guarded method calling another guarded method is one Redis interaction, so only + the outermost admission reports. The inner one leaves its swallowed failures in the + context for the outer to judge, otherwise the outer would read a clean context and + reset the streak the inner just fed. """ + if admission.nested: + return swallowed: Final = _take_swallowed_failures(admission) if admission.generation != breaker.generation: return @@ -459,7 +474,13 @@ def _exit_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmis breaker.record_failure(is_timeout=is_timeout) +def _leave_circuit_breaker() -> None: + _breaker_depth.set(_breaker_depth.get() - 1) + + def _fail_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmission, exc: BaseException) -> None: + if admission.nested: + return swallowed: Final = _take_swallowed_failures(admission) if admission.generation != breaker.generation: return @@ -485,6 +506,8 @@ async def _run_under_circuit_breaker( except Exception as e: _fail_circuit_breaker(breaker, admission, e) raise + finally: + _leave_circuit_breaker() _exit_circuit_breaker(breaker, admission) return result @@ -501,6 +524,8 @@ def _run_under_circuit_breaker_sync( except Exception as e: _fail_circuit_breaker(breaker, admission, e) raise + finally: + _leave_circuit_breaker() _exit_circuit_breaker(breaker, admission) return result @@ -1441,12 +1466,12 @@ class RedisCache(BaseCache): key_value_dict = {} _key_list: Final = [key for key in key_list if key is not None] start_time: Final = time.time() - admission: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") try: _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list] - results: Final = self._run_redis_mget_operation(keys=_keys) - _exit_circuit_breaker(self._circuit_breaker, admission) + results: Final = _run_under_circuit_breaker_sync( + self._circuit_breaker, "batch_get_cache", lambda: self._run_redis_mget_operation(keys=_keys) + ) end_time: Final = time.time() _duration: Final = end_time - start_time self.service_logger_obj.service_success_hook( @@ -1470,6 +1495,8 @@ class RedisCache(BaseCache): decoded_results[k] = v return decoded_results + except RedisCircuitBreakerOpenError: + raise except Exception as e: failed_at: Final = time.time() self.service_logger_obj.service_failure_hook( @@ -1482,7 +1509,6 @@ class RedisCache(BaseCache): parent_otel_span=parent_otel_span, ) verbose_logger.error("Error occurred in batch get cache - %s", e) - _fail_circuit_breaker(self._circuit_breaker, admission, e) return key_value_dict @_redis_circuit_breaker_guard diff --git a/tests/test_litellm/caching/test_caching.py b/tests/test_litellm/caching/test_caching.py index 5d326a9ece0..d135d2892c5 100644 --- a/tests/test_litellm/caching/test_caching.py +++ b/tests/test_litellm/caching/test_caching.py @@ -255,26 +255,41 @@ def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param): @pytest.mark.asyncio -async def test_async_add_cache_treats_an_open_breaker_as_a_quiet_skip(caplog): - """The top-level write wrapper logs a full ERROR traceback for any failure. An open Redis - breaker fails every write instantly, so under load that wrapper alone was hundreds of +@pytest.mark.parametrize("write", ["add_cache", "async_add_cache", "async_add_cache_pipeline"]) +async def test_add_cache_wrappers_treat_an_open_breaker_as_a_quiet_skip(caplog, write: str): + """The top-level write wrappers log a full ERROR traceback for any failure. An open Redis + breaker fails every write instantly, so under load those wrappers alone were hundreds of stack formats per second per replica. Unexpected failures must still get the traceback. """ - from unittest.mock import AsyncMock + from unittest.mock import AsyncMock, MagicMock from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + from litellm.types.utils import Embedding, EmbeddingResponse caplog.set_level(logging.DEBUG, logger="LiteLLM") cache = Cache(type=LiteLLMCacheType.LOCAL) - cache.cache.async_set_cache = AsyncMock(side_effect=RedisCircuitBreakerOpenError("open")) + embedding = EmbeddingResponse(model="text-embedding-3-small", data=[Embedding(embedding=[0.1], index=0, object="embedding")]) - await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + async def run_write() -> None: + if write == "add_cache": + cache.add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + elif write == "async_add_cache": + await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + else: + await cache.async_add_cache_pipeline(embedding, model="text-embedding-3-small", input="hi") + + def install_backend(exc: Exception) -> None: + cache.cache.set_cache = MagicMock(side_effect=exc) + cache.cache.async_set_cache = AsyncMock(side_effect=exc) + cache.cache.async_set_cache_pipeline = AsyncMock(side_effect=exc) + + install_backend(RedisCircuitBreakerOpenError("open")) + await run_write() assert [r.levelno for r in caplog.records if "add_cache" in r.getMessage()] == [logging.DEBUG] - cache.cache.async_set_cache.assert_awaited_once() - cache.cache.async_set_cache = AsyncMock(side_effect=OSError("disk full")) - await cache.async_add_cache("result", model="gpt-5.4-nano", messages=[{"role": "user", "content": "hi"}]) + install_backend(OSError("disk full")) + await run_write() errors = [r for r in caplog.records if r.levelno == logging.ERROR] assert len(errors) == 1 and errors[0].exc_info is not None diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index b2462b297fc..4177c215d0b 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -621,6 +621,11 @@ def dual_cache_with_open_breaker(): pytest.param(lambda c: c.async_set_cache("lit7468", "v"), lambda n: None, id="async_set_cache"), pytest.param(lambda c: c.async_set_cache_pipeline([("lit7468", "v")]), lambda n: None, id="async_set_cache_pipeline"), pytest.param(lambda c: c.async_increment_cache("lit7468", 1.0, ttl=60), float, id="async_increment_cache"), + pytest.param( + lambda c: c.async_increment_cache_pipeline([{"key": "lit7468", "increment_value": 1.0, "ttl": 60}]), + lambda n: [float(n)], + id="async_increment_cache_pipeline", + ), ], ) async def test_open_breaker_is_a_quiet_cache_miss(dual_cache_with_open_breaker, call, expected, caplog): diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 3bc3aa1e047..8c994a4742e 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -494,6 +494,7 @@ def _closed_port() -> int: pytest.param(lambda c: c.async_batch_get_cache(["lit4930"]), id="async_batch_get_cache"), pytest.param(lambda c: c.async_set_cache("lit4930", "v"), id="async_set_cache"), pytest.param(lambda c: c.async_get_ttl("lit4930"), id="async_get_ttl"), + pytest.param(lambda c: c.async_set_cache_sadd("lit4930", ["v"], ttl=None), id="async_set_cache_sadd"), ], ) async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_method): @@ -505,6 +506,7 @@ async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_met breaker could never open. An unreachable Redis then stayed in the pool and every request kept paying the full socket timeout on it. """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD cache = await asyncio.to_thread(RedisCache, host="127.0.0.1", port=_closed_port(), socket_timeout=0.5) @@ -512,10 +514,73 @@ async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_met for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): await call_method(cache) - with pytest.raises(Exception, match="circuit breaker is open"): + with pytest.raises(RedisCircuitBreakerOpenError): await call_method(cache) +@pytest.mark.asyncio +async def test_nested_guarded_flush_failures_still_open_the_breaker(): + """A guarded method that delegates to another guarded method is one Redis call, not two. + + batch_cache_write is guarded and flushes through the guarded async_set_cache_pipeline, + which swallows the pipeline error. If the inner guard consumes that failure, the outer + guard sees a clean run and records a success, so the streak resets on every write and + a dead Redis keeps receiving flushes instead of tripping the breaker. + """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + + cache = await asyncio.to_thread( + RedisCache, host="127.0.0.1", port=_closed_port(), socket_timeout=0.5, redis_flush_size=1 + ) + + for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): + await cache.batch_cache_write("lit7468", "v") + + with pytest.raises(RedisCircuitBreakerOpenError): + await cache.batch_cache_write("lit7468", "v") + assert cache._circuit_breaker._failure_count == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + + +@pytest.mark.asyncio +async def test_nested_guard_that_raises_counts_one_failure_for_the_outer_call(): + """An inner guarded call that raises through the outer one is a single Redis failure, and a + swallowed inner failure followed by an outer raise is two, so the count follows what Redis + actually refused rather than how many guard frames the error crossed. + """ + from redis.exceptions import ConnectionError as RedisConnectionError + + from litellm.caching.redis_cache import ( + RedisCircuitBreaker, + _record_swallowed_redis_failure, + _run_under_circuit_breaker, + ) + + breaker = RedisCircuitBreaker(failure_threshold=10, recovery_timeout=60) + + async def refused(): + raise RedisConnectionError("refused") + + async def outer_delegating_to_inner(): + return await _run_under_circuit_breaker(breaker, "inner", refused) + + async def outer_swallowing_then_raising(): + async def inner_swallowing(): + _record_swallowed_redis_failure(RedisConnectionError("refused")) + return {} + + await _run_under_circuit_breaker(breaker, "inner", inner_swallowing) + raise RedisConnectionError("refused") + + with pytest.raises(RedisConnectionError): + await _run_under_circuit_breaker(breaker, "outer", outer_delegating_to_inner) + assert breaker._failure_count == 1 + + with pytest.raises(RedisConnectionError): + await _run_under_circuit_breaker(breaker, "outer", outer_swallowing_then_raising) + assert breaker._failure_count == 3 + + def test_sync_batch_get_cache_swallowed_failures_open_the_breaker_and_then_fast_fail(sync_batch_redis_cache): """The sync batch read hides its Redis error behind an empty dict, but the breaker must still count it, and once open the read has to raise the typed error like its async twin so DualCache @@ -1106,12 +1171,13 @@ async def test_success_admitted_before_the_breaker_opened_cannot_close_it(): @pytest.mark.asyncio -async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay_recovery(): - """A stale in-flight call that swallows its Redis error must not restart the open timer. +@pytest.mark.parametrize("stale_call_raises", [False, True], ids=["swallows", "raises"]) +async def test_failure_admitted_before_the_breaker_opened_cannot_delay_recovery(stale_call_raises: bool): + """A stale in-flight call that fails after the breaker opened must not restart the open timer. - Guarded methods that return a default instead of raising still report their failure to - the breaker on exit. If that report landed against a newer breaker generation, every - slow call that was already dialing Redis when the breaker opened would push recovery + Whether the method swallows its Redis error and returns a default or lets it propagate, + the failure is reported on exit. If that report landed against a newer breaker generation, + every slow call that was already dialing Redis when the breaker opened would push recovery back by its own socket timeout, and knock out the HALF_OPEN probe if it landed then. """ from redis.exceptions import ConnectionError as RedisConnectionError @@ -1125,8 +1191,10 @@ async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay breaker = RedisCircuitBreaker(failure_threshold=2, recovery_timeout=0.05) release_stale_call = asyncio.Event() - async def stale_call_that_swallows_its_failure(): + async def stale_call(): await release_stale_call.wait() + if stale_call_raises: + raise RedisConnectionError("refused") _record_swallowed_redis_failure(RedisConnectionError("refused")) return {} @@ -1136,7 +1204,7 @@ async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay async def recovered(): return "ok" - in_flight = asyncio.create_task(_run_under_circuit_breaker(breaker, "stale", stale_call_that_swallows_its_failure)) + in_flight = asyncio.create_task(_run_under_circuit_breaker(breaker, "stale", stale_call)) await asyncio.sleep(0) for _ in range(breaker.failure_threshold): with pytest.raises(RedisConnectionError): @@ -1145,7 +1213,11 @@ async def test_swallowed_failure_admitted_before_the_breaker_opened_cannot_delay await asyncio.sleep(0.04) release_stale_call.set() - assert await in_flight == {} + if stale_call_raises: + with pytest.raises(RedisConnectionError): + await in_flight + else: + assert await in_flight == {} await asyncio.sleep(0.03) assert await _run_under_circuit_breaker(breaker, "probe", recovered) == "ok" From 71d1bfb70a7f9ad2cac9e4e2090f029791fbd2e7 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 15:55:03 -0700 Subject: [PATCH 379/411] feat(ui): link the User, Team and Created By cells on the Virtual Keys page The key detail page already walks out to the user, team and org behind a key, but the Virtual Keys table rendered those same values as dead text, so getting to a team meant copying its alias and searching the Teams page. User, Team and Created By now render through the shared IdentityCell with an href, the same hover-highlight-and-chevron affordance the Key column already uses. Sentinel ids do not get a link, since they have no detail page to open. Rather than repeat that check at every call site, teamDetailHref and userDetailHref now return undefined for "litellm-dashboard" and "default_user_id", the way modelGroupHref already does for model grants, and EntityLink falls back to plain text when it has no href, the way BadgeLink already does. Both sentinels move into src/utils/sentinels.ts instead of staying as string literals scattered across components. Claude-Session: https://claude.ai/code/session_01NfwfQhamRNnSqgXMUjf3h4 --- .../_components/AccessGroupsDetailsPage.tsx | 2 +- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 71 +++++++++++++++++++ .../VirtualKeysPage/keyTableColumns.tsx | 45 ++++++------ .../DefaultProxyAdminTag.tsx | 5 +- .../common_components/LabeledField.tsx | 3 +- .../src/components/shared/EntityLink.test.tsx | 6 ++ .../src/components/shared/EntityLink.tsx | 14 +++- .../src/utils/entityLinks.test.ts | 24 ++++++- ui/litellm-dashboard/src/utils/entityLinks.ts | 7 +- ui/litellm-dashboard/src/utils/sentinels.ts | 3 + 10 files changed, 147 insertions(+), 33 deletions(-) create mode 100644 ui/litellm-dashboard/src/utils/sentinels.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx index 1eeebe4ebba..118a8655d5a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx @@ -54,7 +54,7 @@ function ResourceBadge({ fallback, }: { resource: AccessGroupResource; - href: string; + href?: string; fallback: (id: string) => string; }) { const badge = ( diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 881b0b93ff9..57786d71c66 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -473,6 +473,77 @@ it("should display 'Default Proxy Admin' for user_id when value is 'default_user }); }); +describe("entity links out of the key rows", () => { + const keyRow = async () => (await screen.findByText("Test Key Alias")).closest("tr") as HTMLElement; + + const enableCreatedByColumn = async (user: ReturnType) => { + await user.click(screen.getByRole("button", { name: "Columns" })); + await user.click(await screen.findByText("Created By")); + await user.keyboard("{Escape}"); + }; + + it("points the User and Team cells at their detail pages", async () => { + renderWithProviders(); + + const row = await keyRow(); + expect(within(row).getByRole("link", { name: "user@example.com" })).toHaveAttribute( + "href", + "/ui/users?user=user-1", + ); + expect(within(row).getByRole("link", { name: "Test Team" })).toHaveAttribute("href", "/ui/teams?team=team-1"); + }); + + it("points the Created By cell at the creator's detail page", async () => { + mockUseKeys.mockReturnValue( + keysResult([ + { + ...mockKey, + created_by: "creator-1", + created_by_user: { user_id: "creator-1", user_email: "creator@example.com", user_alias: "The Creator" }, + }, + ]), + ); + const user = userEvent.setup(); + renderWithProviders(); + await enableCreatedByColumn(user); + + const row = await keyRow(); + expect(within(row).getByRole("link", { name: "The Creator" })).toHaveAttribute("href", "/ui/users?user=creator-1"); + }); + + it("leaves the default_user_id placeholder unlinked even once it resolves to a named user", async () => { + const placeholder = { user_id: "default_user_id", user_email: "admin@example.com", user_alias: "Proxy Admin" }; + mockUseKeys.mockReturnValue( + keysResult([ + { + ...mockKey, + user_id: placeholder.user_id, + user_email: placeholder.user_email, + user: placeholder, + created_by: placeholder.user_id, + created_by_user: placeholder, + }, + ]), + ); + const user = userEvent.setup(); + renderWithProviders(); + await enableCreatedByColumn(user); + + const row = await keyRow(); + expect(within(row).getAllByText("Proxy Admin")).toHaveLength(2); + expect(within(row).queryByRole("link", { name: "Proxy Admin" })).not.toBeInTheDocument(); + }); + + it("leaves the litellm-dashboard session team unlinked", async () => { + mockUseKeys.mockReturnValue(keysResult([{ ...mockKey, team_id: "litellm-dashboard" }])); + renderWithProviders(); + + const row = await keyRow(); + expect(within(row).getByText("litellm-dashboard")).toBeInTheDocument(); + expect(within(row).queryByRole("link", { name: "litellm-dashboard" })).not.toBeInTheDocument(); + }); +}); + it("should render table without crashing when models is null", async () => { mockUseKeys.mockReturnValue(keysResult([{ ...mockKey, models: null as unknown as string[] }])); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index 0865fe76519..c87bc90fb47 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -16,6 +16,8 @@ import { StatusBadge, type StatusTone, } from "@/components/shared/table_cells"; +import { teamDetailHref, userDetailHref } from "@/utils/entityLinks"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; @@ -27,6 +29,8 @@ interface KeyStatus { tooltip?: string; } +const ENTITY_CELL_TITLE_CLASSES = "font-mono text-xs font-normal"; + const SPEND_BUDGET_SORT_FIELDS: DataTableSortField[] = [ { id: "spend", label: "Spend" }, { id: "max_budget", label: "Budget" }, @@ -74,7 +78,7 @@ const UserPopoverCell = ({ width: number; }) => { const displayValue = userAlias || userEmail || userId; - const isDefaultAdmin = userId === "default_user_id"; + const isDefaultAdmin = userId === DEFAULT_PROXY_ADMIN_USER_ID; const popoverContent = (
@@ -95,28 +99,21 @@ const UserPopoverCell = ({
); - if (isDefaultAdmin && !userAlias && !userEmail) { - return ( - - }> - - - {popoverContent} - + const trigger = + isDefaultAdmin && !userAlias && !userEmail ? ( + + ) : ( + ); - } return ( - - } - > - {displayValue || "-"} + }> + {trigger} {popoverContent} @@ -201,12 +198,12 @@ export const getKeyTableColumns = ({ const teamId = info.getValue() as string | null; if (!teamId) return "-"; const team = allTeams.find((t) => t.team_id === teamId); - const displayValue = team?.team_alias || teamId; - const width = info.cell.column.getSize(); return ( - - {displayValue} - + ); }, }, diff --git a/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx b/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx index 9ec24bb929b..308f722a8f2 100644 --- a/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx +++ b/ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx @@ -1,13 +1,12 @@ import { Badge } from "@/components/ui/badge"; - -const DEFAULT_USER_ID = "default_user_id"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; interface DefaultProxyAdminTagProps { userId: string | null | undefined; } export default function DefaultProxyAdminTag({ userId }: DefaultProxyAdminTagProps) { - if (userId === DEFAULT_USER_ID) { + if (userId === DEFAULT_PROXY_ADMIN_USER_ID) { return Default Proxy Admin; } diff --git a/ui/litellm-dashboard/src/components/common_components/LabeledField.tsx b/ui/litellm-dashboard/src/components/common_components/LabeledField.tsx index 9f45b05f306..99046cfa84d 100644 --- a/ui/litellm-dashboard/src/components/common_components/LabeledField.tsx +++ b/ui/litellm-dashboard/src/components/common_components/LabeledField.tsx @@ -2,6 +2,7 @@ import React from "react"; import CopyButton from "@/components/shared/CopyButton"; import { EntityLink } from "@/components/shared/EntityLink"; import { cx } from "@/lib/cva.config"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; import DefaultProxyAdminTag from "./DefaultProxyAdminTag"; interface LabeledFieldProps { @@ -24,7 +25,7 @@ export default function LabeledField({ defaultUserIdCheck = false, }: LabeledFieldProps) { const isEmpty = !value; - const isDefaultUser = defaultUserIdCheck && value === "default_user_id"; + const isDefaultUser = defaultUserIdCheck && value === DEFAULT_PROXY_ADMIN_USER_ID; const displayValue = isEmpty ? "-" : value; const isCopyable = copyable && !isEmpty && !isDefaultUser; const isLink = href != null && !isEmpty && !isDefaultUser; diff --git a/ui/litellm-dashboard/src/components/shared/EntityLink.test.tsx b/ui/litellm-dashboard/src/components/shared/EntityLink.test.tsx index a02f1699c30..3d6fc9a435b 100644 --- a/ui/litellm-dashboard/src/components/shared/EntityLink.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/EntityLink.test.tsx @@ -25,6 +25,12 @@ describe("EntityLink", () => { expect(push).toHaveBeenCalledWith("/ui/users?user=u1"); }); + it("renders the label as plain text when there is no href to point at", () => { + render(default_user_id); + expect(screen.queryByRole("link")).not.toBeInTheDocument(); + expect(screen.getByText("default_user_id")).toBeInTheDocument(); + }); + it("leaves modified clicks to the browser so new-tab shortcuts keep working", async () => { const user = userEvent.setup(); render(alice); diff --git a/ui/litellm-dashboard/src/components/shared/EntityLink.tsx b/ui/litellm-dashboard/src/components/shared/EntityLink.tsx index 4054b929943..4885835a42c 100644 --- a/ui/litellm-dashboard/src/components/shared/EntityLink.tsx +++ b/ui/litellm-dashboard/src/components/shared/EntityLink.tsx @@ -19,12 +19,24 @@ export function useEntityLinkClick(href: string): (e: React.MouseEvent) => void } interface EntityLinkProps { - href: string; + href?: string; className?: string; children: React.ReactNode; } export function EntityLink({ href, className, children }: EntityLinkProps) { + if (!href) { + return {children}; + } + + return ( + + {children} + + ); +} + +function LinkedEntity({ href, className, children }: EntityLinkProps & { href: string }) { const handleClick = useEntityLinkClick(href); return ( diff --git a/ui/litellm-dashboard/src/utils/entityLinks.test.ts b/ui/litellm-dashboard/src/utils/entityLinks.test.ts index 47161a903ed..231413b213a 100644 --- a/ui/litellm-dashboard/src/utils/entityLinks.test.ts +++ b/ui/litellm-dashboard/src/utils/entityLinks.test.ts @@ -2,7 +2,29 @@ import { describe, expect, it, vi } from "vitest"; vi.mock("@/components/networking", () => ({ serverRootPath: "" })); -import { modelGroupHref } from "./entityLinks"; +import { modelGroupHref, teamDetailHref, userDetailHref } from "./entityLinks"; + +describe("userDetailHref", () => { + it("targets the users page filtered to the encoded user id", () => { + expect(userDetailHref("user-1")).toMatch(/\/users\?user=user-1$/); + expect(userDetailHref("a b/c")).toMatch(/\?user=a%20b%2Fc$/); + }); + + it("returns no href for the proxy admin placeholder, which has no user page", () => { + expect(userDetailHref("default_user_id")).toBeUndefined(); + }); +}); + +describe("teamDetailHref", () => { + it("targets the teams page filtered to the encoded team id", () => { + expect(teamDetailHref("team-1")).toMatch(/\/teams\?team=team-1$/); + expect(teamDetailHref("a b/c")).toMatch(/\?team=a%20b%2Fc$/); + }); + + it("returns no href for the Admin UI session team, which has no team page", () => { + expect(teamDetailHref("litellm-dashboard")).toBeUndefined(); + }); +}); describe("modelGroupHref", () => { it("targets the models page filtered to the encoded model group", () => { diff --git a/ui/litellm-dashboard/src/utils/entityLinks.ts b/ui/litellm-dashboard/src/utils/entityLinks.ts index b8c70bdda48..33f2aa34976 100644 --- a/ui/litellm-dashboard/src/utils/entityLinks.ts +++ b/ui/litellm-dashboard/src/utils/entityLinks.ts @@ -1,3 +1,4 @@ +import { DEFAULT_PROXY_ADMIN_USER_ID, UI_TEAM_ID } from "@/utils/sentinels"; import { uiHref } from "@/utils/uiHref"; const MODEL_GRANT_SENTINELS: ReadonlySet = new Set([ @@ -6,7 +7,8 @@ const MODEL_GRANT_SENTINELS: ReadonlySet = new Set([ "no-default-models", ]); -export function teamDetailHref(teamId: string): string { +export function teamDetailHref(teamId: string): string | undefined { + if (teamId === UI_TEAM_ID) return undefined; return `${uiHref("teams")}?team=${encodeURIComponent(teamId)}`; } @@ -14,7 +16,8 @@ export function keyDetailHref(keyToken: string): string { return `${uiHref("api-keys")}?key=${encodeURIComponent(keyToken)}`; } -export function userDetailHref(userId: string): string { +export function userDetailHref(userId: string): string | undefined { + if (userId === DEFAULT_PROXY_ADMIN_USER_ID) return undefined; return `${uiHref("users")}?user=${encodeURIComponent(userId)}`; } diff --git a/ui/litellm-dashboard/src/utils/sentinels.ts b/ui/litellm-dashboard/src/utils/sentinels.ts new file mode 100644 index 00000000000..da6d09dfdd7 --- /dev/null +++ b/ui/litellm-dashboard/src/utils/sentinels.ts @@ -0,0 +1,3 @@ +export const DEFAULT_PROXY_ADMIN_USER_ID = "default_user_id"; + +export const UI_TEAM_ID = "litellm-dashboard"; From 311441ced19b1963f8ad22b758a8ae6171105a74 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 10 Sep 2026 16:08:57 -0700 Subject: [PATCH 380/411] bump: litellm-proxy-extras 0.4.95 -> 0.4.96 --- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 2 +- uv.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 91b4e4a7ba1..7d4c78088f1 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.95" +version = "0.4.96" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.95" +version = "0.4.96" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 04f2f3fd1dd..d33d693f794 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.95", + "litellm-proxy-extras==0.4.96", "litellm-enterprise==0.1.66", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", diff --git a/uv.lock b/uv.lock index 0fe787645a2..cedc505acdf 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-09-06T00:40:30.433549Z" +exclude-newer = "2026-09-07T23:09:03.362777Z" exclude-newer-span = "P3D" [manifest] @@ -4772,7 +4772,7 @@ source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.95" +version = "0.4.96" source = { editable = "litellm-proxy-extras" } [[package]] From 3b93da414ead03604bb9dcba0b24be585db28639 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:18:00 -0700 Subject: [PATCH 381/411] feat(proxy): let the in-container pgbouncer follow rotating RDS IAM and Azure Entra tokens (#40623) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- gateway/launch.py | 20 +- helm/litellm/templates/_helpers.tpl | 5 +- helm/litellm/tests/connection_pool_tests.yaml | 28 +- helm/litellm/values.yaml | 6 +- litellm/proxy/db/db_url_settings.py | 5 + litellm/proxy/db/pgbouncer.py | 280 +++++++++++++--- litellm/proxy/proxy_cli.py | 12 +- litellm/proxy/utils.py | 6 +- terraform/litellm/aws/README.md | 11 +- terraform/litellm/aws/ecs.tf | 4 +- .../aws/tests/connection_pool.tftest.hcl | 12 +- terraform/litellm/aws/variables.tf | 7 +- tests/test_gateway/test_launch.py | 75 ++++- tests/test_litellm/proxy/db/test_pgbouncer.py | 314 ++++++++++++++++-- tests/test_litellm/proxy/test_proxy_utils.py | 49 ++- 15 files changed, 703 insertions(+), 131 deletions(-) diff --git a/gateway/launch.py b/gateway/launch.py index 6b28fafdcf6..d67432caee1 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -4,8 +4,9 @@ is fine for a plain Postgres URL but not for the pooler: PgBouncer must be started exactly once per pod, before the workers fork, and the workers must be handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in -``DatabaseURLSettings.apply_to_env`` under password auth, so setting it here is -enough for every worker to pick the pooled URL up unchanged. +``DatabaseURLSettings.apply_to_env`` under password auth, and one marked pooled +wins under token auth too, so exporting it here is enough for every worker to +pick the pooled URL up unchanged. Run with: python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000 @@ -19,7 +20,12 @@ from typing import Final from uvicorn.main import main as uvicorn_main from litellm.proxy.db.db_url_settings import DatabaseURLSettings -from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_in_container_pgbouncer +from litellm.proxy.db.pgbouncer import ( + PgBouncerError, + PgBouncerSettings, + export_pooled_database_url, + start_in_container_pgbouncer, +) GATEWAY_APP: Final = "gateway.main:app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" @@ -41,15 +47,15 @@ def pool_database_url( """Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off. The upstream URL is whatever ``apply_to_env`` assembled from the discrete - ``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Token auth is - rejected by the pooler itself, since it holds one password for its lifetime. + ``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token + auth the pooler mints and renews the upstream token itself. """ if not pgbouncer.enabled: return None upstream_url: Final = environ.get("DATABASE_URL") if upstream_url is None: return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled") - return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth_enabled=settings.token_auth() is not None) + return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) def _serve(argv: Sequence[str]) -> None: @@ -63,7 +69,7 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) - if isinstance(pooled_url, PgBouncerError): sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") if pooled_url is not None: - os.environ["DATABASE_URL"] = pooled_url + export_pooled_database_url(pooled_url) serve(uvicorn_argv(argv, os.environ)) diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 4ad3cfe0484..dc47bc0b0a1 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -361,12 +361,9 @@ harmless no-op for the Job and authoritative for the app pods. {{- end -}} {{/* -In-container PgBouncer env for the gateway container. Fails at render time under IAM or Entra auth: the pooler holds one static password for the life of the pod. +In-container PgBouncer env for the gateway container. Under IAM or Entra auth the pooler mints and renews the database token itself. */}} {{- define "litellm.connectionPoolEnv" -}} -{{- if or .Values.database.writer.useIAMAuth .Values.database.writer.useAzureEntraAuth }} -{{- fail "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" }} -{{- end }} {{- with .Values.database.connectionPool -}} - name: LITELLM_PGBOUNCER_ENABLED value: "true" diff --git a/helm/litellm/tests/connection_pool_tests.yaml b/helm/litellm/tests/connection_pool_tests.yaml index 31be7575f3c..6dd9c274a8c 100644 --- a/helm/litellm/tests/connection_pool_tests.yaml +++ b/helm/litellm/tests/connection_pool_tests.yaml @@ -82,23 +82,39 @@ tests: name: LITELLM_PGBOUNCER_ENABLED any: true - - it: pool with IAM auth fails at render time + - it: pool with IAM auth renders both the pool and the token auth flag template: gateway/deployment.yaml set: database.connectionPool.enabled: true database.writer.useIAMAuth: true asserts: - - failedTemplate: - errorMessage: "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: IAM_TOKEN_DB_AUTH + value: "true" - - it: pool with Entra auth fails at render time + - it: pool with Entra auth renders both the pool and the token auth flag template: gateway/deployment.yaml set: database.connectionPool.enabled: true database.writer.useAzureEntraAuth: true asserts: - - failedTemplate: - errorMessage: "database.connectionPool.enabled cannot be combined with database.writer.useIAMAuth or database.writer.useAzureEntraAuth: the in-container pgbouncer holds a static database password and cannot follow a rotating token. Disable the pool or use a static database password" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: AZURE_POSTGRESQL_AUTH + value: "true" - it: IAM auth without the pool still renders template: gateway/deployment.yaml diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 45ab5229f38..677e21a5a51 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -235,9 +235,9 @@ database: # network hop. The chart emits LITELLM_PGBOUNCER_ENABLED / # LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / LITELLM_PGBOUNCER_MAX_CLIENT_CONN on # the gateway container only: the backend runs a single worker and the - # migrations Job must keep a direct connection. The pool holds a static - # password, so it cannot be combined with `database.writer.useIAMAuth` or - # `useAzureEntraAuth` (rendering fails). Starting profile for + # migrations Job must keep a direct connection. With + # `database.writer.useIAMAuth` or `useAzureEntraAuth` the pool mints and + # renews the database token itself, so the workers never see it. Starting profile for # `gateway.numWorkers: 4` is maxDbConnections: 20, so a database with a # 5000-connection ceiling fits roughly 200 gateway replicas. connectionPool: diff --git a/litellm/proxy/db/db_url_settings.py b/litellm/proxy/db/db_url_settings.py index d1e4b3e92b8..e93bc96da69 100644 --- a/litellm/proxy/db/db_url_settings.py +++ b/litellm/proxy/db/db_url_settings.py @@ -52,6 +52,7 @@ from typing import Annotated, Final, Protocol, TypeAlias, cast from pydantic import AliasChoices, BeforeValidator, Field from pydantic_settings import BaseSettings, SettingsConfigDict +from litellm.proxy.db.pgbouncer import database_url_is_pooled from litellm.proxy.db.token_auth import ( AZURE_POSTGRESQL_AUTH_ENV_VAR, DEFAULT_POSTGRES_PORT, @@ -358,8 +359,12 @@ class DatabaseURLSettings(BaseSettings): Raises ``RuntimeError`` (naming the offending vars) when token auth is enabled but a required field is missing — the proxy cannot recover from this and a clear startup error beats a Prisma connect failure. + A ``DATABASE_URL`` the supervisor pointed at the in-container PgBouncer + is kept even under token auth: the pooler renews the token upstream. """ auth: Final = self.token_auth() + if auth is not None and database_url_is_pooled(): + return None if auth is not None: missing: Final = tuple( env diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py index 7792867600c..eac80f7a712 100644 --- a/litellm/proxy/db/pgbouncer.py +++ b/litellm/proxy/db/pgbouncer.py @@ -18,19 +18,28 @@ Migrations and the schema diff run in the supervisor before the pooler is started, so they always go straight to Postgres. ``DATABASE_URL_READ_REPLICA`` is left untouched. -The pooler holds the database password from startup, so it cannot be combined -with ``IAM_TOKEN_DB_AUTH`` or ``AZURE_POSTGRESQL_AUTH``: those rotate the -password inside every worker on their own schedule, and PgBouncer would keep -authenticating upstream with the expired token. +The workers never hold the upstream credential: they log in to PgBouncer as +``litellm_pgbouncer`` with a random password made at startup, and PgBouncer +takes the database user's password from its auth file. Under +``IAM_TOKEN_DB_AUTH`` or ``AZURE_POSTGRESQL_AUTH`` that password is a +short-lived token, so the supervisor mints a new one before it expires, +rewrites the auth file and asks PgBouncer to reload; only new upstream +connections authenticate, so live ones are unaffected. The pooled +``DATABASE_URL`` then carries a static password, and the workers must not run +their own token refresh against it: ``LITELLM_PGBOUNCER_POOLED_DATABASE_URL`` +tells them so, while a read replica keeps refreshing its own token. """ from __future__ import annotations import atexit +import functools import os import re +import secrets import shlex import shutil +import signal import socket import subprocess import tempfile @@ -39,6 +48,7 @@ import time import urllib.parse from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from types import MappingProxyType from typing import Final @@ -47,10 +57,18 @@ from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict from litellm._logging import verbose_proxy_logger -from litellm.proxy.db.token_auth import AZURE_POSTGRESQL_AUTH_ENV_VAR, IAM_TOKEN_DB_AUTH_ENV_VAR +from litellm.proxy.db.token_auth import ( + DatabaseTokenAuth, + IAMEndpoint, + mint_database_token, + parse_database_token_expiration, + parse_iam_endpoint_from_url, +) PGBOUNCER_ENV_PREFIX: Final = "LITELLM_PGBOUNCER_" +PGBOUNCER_POOLED_ENV_VAR: Final = "LITELLM_PGBOUNCER_POOLED_DATABASE_URL" PGBOUNCER_LISTEN_ADDR: Final = "127.0.0.1" +PGBOUNCER_POOL_USER: Final = "litellm_pgbouncer" PGBOUNCER_INI_NAME: Final = "pgbouncer.ini" PGBOUNCER_USERLIST_NAME: Final = "userlist.txt" PGBOUNCER_CA_NAME: Final = "server-ca.pem" @@ -59,13 +77,11 @@ PGBOUNCER_READY_TIMEOUT_SECONDS: Final = 15.0 PGBOUNCER_STOP_GRACE_SECONDS: Final = 10.0 PGBOUNCER_UNPRIVILEGED_USER: Final = "nobody" PGBOUNCER_MIN_VERSION: Final = (1, 19) +PGBOUNCER_MAX_PASSWORD_BYTES: Final = 2048 PGBOUNCER_VERSION_PATTERN: Final = re.compile(r"PgBouncer (\d+)\.(\d+)") -PGBOUNCER_LIST_DELIMITER_PATTERN: Final = re.compile(r"[,\s]") -PGBOUNCER_TOKEN_AUTH_CONFLICT: Final = ( - f"the in-container pgbouncer cannot be combined with {IAM_TOKEN_DB_AUTH_ENV_VAR} or " - f"{AZURE_POSTGRESQL_AUTH_ENV_VAR}: each worker rotates the database password on its own schedule and the pooler " - "would keep using the expired token upstream. Disable the pooler or use a static database password" -) +PGBOUNCER_TOKEN_REFRESH_BUFFER_SECONDS: Final = 180.0 +PGBOUNCER_TOKEN_FALLBACK_REFRESH_SECONDS: Final = 600.0 +PGBOUNCER_TOKEN_RETRY_SECONDS: Final = 30.0 # Prisma's client-side TLS params describe the hop to Postgres, which becomes # PgBouncer's server side. They move into ``server_tls_*`` and must not stay on @@ -97,10 +113,18 @@ class PgBouncerSettings(BaseSettings): @dataclass(frozen=True, slots=True) class PgBouncerPlan: ini: str - userlist: str pooled_url: str + upstream_user: str + upstream_password: str | None + pool_password: str ca_source: str | None = None + def userlist(self, upstream_password: str) -> str: + return "".join( + f"{_userlist_quote(user)} {_userlist_quote(password)}\n" + for user, password in ((self.upstream_user, upstream_password), (PGBOUNCER_POOL_USER, self.pool_password)) + ) + @dataclass(frozen=True, slots=True) class PgBouncerError: @@ -173,7 +197,9 @@ def plan_pgbouncer( Params describing Prisma's own pool (``connection_limit``, ``pool_timeout``, ...) stay on the pooled URL; the TLS params and ``options`` describe the hop - to Postgres and move into the PgBouncer config. ``run_as_user`` is the + to Postgres and move into the PgBouncer config. The upstream password is + left out of the config on purpose: PgBouncer then takes it from the auth + file, which can be rewritten while it runs. ``run_as_user`` is the unprivileged user PgBouncer drops to when the proxy runs as root, which PgBouncer itself refuses to do. """ @@ -184,14 +210,12 @@ def plan_pgbouncer( dbname: Final = urllib.parse.unquote(parsed.path.lstrip("/")) username: Final = urllib.parse.unquote(parsed.username or "") password: Final = None if parsed.password is None else urllib.parse.unquote(parsed.password) - if not parsed.hostname or not username or password is None or not dbname: + if not parsed.hostname or not username or not dbname: + return PgBouncerError("DATABASE_URL must carry a host, user and database name for the in-container PgBouncer") + if username == PGBOUNCER_POOL_USER: return PgBouncerError( - "DATABASE_URL must carry a host, user, password and database name for the in-container PgBouncer" - ) - if PGBOUNCER_LIST_DELIMITER_PATTERN.search(username): - return PgBouncerError( - f"the database user {username!r} cannot be named in PgBouncer's stats_users list: " - "PgBouncer splits list settings on commas and whitespace and has no quoting for them" + f"the database user cannot be named {PGBOUNCER_POOL_USER!r}: that is the user the workers log in to the " + "in-container PgBouncer as, and PgBouncer keeps one password per user" ) if "sslidentity" in params: return PgBouncerError("client certificates (sslidentity) are not supported with the in-container PgBouncer") @@ -212,7 +236,6 @@ def plan_pgbouncer( f"port={parsed.port or 5432}", f"dbname={_single_quoted(dbname)}", f"user={_single_quoted(username)}", - f"password={_single_quoted(password)}", *((f"connect_query={_single_quoted(connect_query)}",) if connect_query else ()), ) ) @@ -227,7 +250,7 @@ def plan_pgbouncer( f"unix_socket_dir = {runtime_dir}", f"auth_file = {runtime_dir / PGBOUNCER_USERLIST_NAME}", "auth_type = scram-sha-256", - f"stats_users = {username}", + f"stats_users = {PGBOUNCER_POOL_USER}", "pool_mode = transaction", f"max_client_conn = {settings.max_client_conn}", f"default_pool_size = {settings.max_db_connections}", @@ -238,41 +261,175 @@ def plan_pgbouncer( "", ) ) - userlist: Final = f"{_userlist_quote(username)} {_userlist_quote(password)}\n" pooled_query: Final = urllib.parse.urlencode( (*((key, value) for key, value in params.items() if key not in POOLED_URL_DROPPED_KEYS), ("pgbouncer", "true")) ) - credentials: Final = f"{urllib.parse.quote(username, safe='')}:{urllib.parse.quote(password, safe='')}" + pool_password: Final = secrets.token_urlsafe(32) pooled_url: Final = urllib.parse.urlunsplit( - parsed._replace(netloc=f"{credentials}@{PGBOUNCER_LISTEN_ADDR}:{settings.port}", query=pooled_query) + parsed._replace( + netloc=f"{PGBOUNCER_POOL_USER}:{pool_password}@{PGBOUNCER_LISTEN_ADDR}:{settings.port}", query=pooled_query + ) + ) + return PgBouncerPlan( + ini=ini, + pooled_url=pooled_url, + upstream_user=username, + upstream_password=password, + pool_password=pool_password, + ca_source=params.get("sslcert") or None, ) - return PgBouncerPlan(ini=ini, userlist=userlist, pooled_url=pooled_url, ca_source=params.get("sslcert") or None) -def write_pgbouncer_files(plan: PgBouncerPlan, runtime_dir: Path, run_as_user: str | None) -> Path | PgBouncerError: - """Write the ini, userlist (both hold the password, so mode 0600) and CA copy, and return the ini path. +def _write_private(path: Path, content: str, run_as_user: str | None) -> None: + with open(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w", encoding="utf-8") as handle: + handle.write(content) + if run_as_user is not None: + shutil.chown(path, user=run_as_user) + + +def write_pgbouncer_ini(plan: PgBouncerPlan, runtime_dir: Path, run_as_user: str | None) -> Path | PgBouncerError: + """Write the ini (mode 0600) and the CA copy, and return the ini path. ``run_as_user`` is the user PgBouncer drops to when started as root; it has to own the files it re-reads on reload and the socket directory. """ ini_path: Final = runtime_dir / PGBOUNCER_INI_NAME - userlist_path: Final = runtime_dir / PGBOUNCER_USERLIST_NAME ca_path: Final = runtime_dir / PGBOUNCER_CA_NAME if plan.ca_source is not None: try: shutil.copyfile(plan.ca_source, ca_path) except OSError as error: return PgBouncerError(f"cannot read the CA bundle {plan.ca_source!r} named by sslcert: {error}") - for path, content in ((userlist_path, plan.userlist), (ini_path, plan.ini)): - path.touch(mode=0o600) - path.write_text(content, encoding="utf-8") + _write_private(ini_path, plan.ini, run_as_user) if run_as_user is not None: runtime_dir.chmod(0o700) - for path in (runtime_dir, ini_path, userlist_path, *((ca_path,) if plan.ca_source is not None else ())): + for path in (runtime_dir, *((ca_path,) if plan.ca_source is not None else ())): shutil.chown(path, user=run_as_user) return ini_path +def write_userlist(userlist: str, runtime_dir: Path, run_as_user: str | None) -> Path: + """Replace the auth file in one step, so a PgBouncer starting or reloading meanwhile reads the old or the new one whole.""" + userlist_path: Final = runtime_dir / PGBOUNCER_USERLIST_NAME + staged_path: Final = runtime_dir / f".{PGBOUNCER_USERLIST_NAME}.next" + _write_private(staged_path, userlist, run_as_user) + os.replace(staged_path, userlist_path) + return userlist_path + + +def export_pooled_database_url(pooled_url: str) -> None: + os.environ["DATABASE_URL"] = pooled_url + os.environ[PGBOUNCER_POOLED_ENV_VAR] = "true" + + +def database_url_is_pooled(environ: Mapping[str, str] = os.environ) -> bool: + return environ.get(PGBOUNCER_POOLED_ENV_VAR) == "true" + + +@dataclass(frozen=True, slots=True) +class PgBouncerTokenSource: + auth: DatabaseTokenAuth + endpoint: IAMEndpoint + + def mint(self) -> str: + """The token as Postgres expects it: ``mint_database_token`` returns it percent-encoded for a URL.""" + return urllib.parse.unquote(mint_database_token(self.auth, self.endpoint)) + + def expires_at(self, token: str) -> datetime | None: + return parse_database_token_expiration(self.auth, token) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None) + + +class PgBouncerTokenRefresher: + """Keeps the token in PgBouncer's auth file current from a daemon thread. + + ``install`` gets each fresh token and is expected to rewrite the auth file + and reload PgBouncer. The next refresh is due ``buffer_seconds`` before the + token expires, or ``fallback_seconds`` later when the expiry cannot be read. + A refresh that fails leaves the previous auth file in place and is retried + after ``retry_seconds``: the old token stays good until it expires, so a + transient credential-provider error costs nothing unless it persists. + """ + + def __init__( + self, + source: PgBouncerTokenSource, + install: Callable[[str], None], + *, + buffer_seconds: float = PGBOUNCER_TOKEN_REFRESH_BUFFER_SECONDS, + fallback_seconds: float = PGBOUNCER_TOKEN_FALLBACK_REFRESH_SECONDS, + retry_seconds: float = PGBOUNCER_TOKEN_RETRY_SECONDS, + now: Callable[[], datetime] = _utcnow, + ) -> None: + self._source: Final = source + self._install: Final = install + self._buffer_seconds: Final = buffer_seconds + self._fallback_seconds: Final = fallback_seconds + self._retry_seconds: Final = retry_seconds + self._now: Final = now + self._stopping: Final = threading.Event() + self._delay: float = 0.0 + self._thread: threading.Thread | None = None + + def refresh(self) -> float | PgBouncerError: + label: Final = self._source.auth.label + try: + token: Final = self._source.mint() + except Exception as mint_error: + return PgBouncerError(f"could not mint a {label} for the in-container pgbouncer: {mint_error!r}") + if len(token.encode()) >= PGBOUNCER_MAX_PASSWORD_BYTES: + return PgBouncerError( + f"the {label} is {len(token.encode())} bytes long, but PgBouncer's auth file holds passwords of at " + f"most {PGBOUNCER_MAX_PASSWORD_BYTES - 1} bytes" + ) + try: + self._install(token) + except OSError as install_error: + return PgBouncerError(f"could not install the {label} into the pgbouncer auth file: {install_error}") + expires_at: Final = self._source.expires_at(token) + if expires_at is None: + return self._fallback_seconds + return max(self._retry_seconds, (expires_at - self._now()).total_seconds() - self._buffer_seconds) + + def start(self) -> PgBouncerError | None: + primed: Final = self.refresh() + if isinstance(primed, PgBouncerError): + return primed + self._delay = primed + self._thread = threading.Thread(target=self._run, daemon=True, name="litellm-pgbouncer-token-refresh") + self._thread.start() + return None + + def _run(self) -> None: + while not self._stopping.wait(self._delay): + self._delay = self._refresh_and_report() + + def _refresh_and_report(self) -> float: + outcome: Final = self.refresh() + if isinstance(outcome, PgBouncerError): + verbose_proxy_logger.error( + "In-container pgbouncer keeps its current %s (%s); retrying in %.0fs.", + self._source.auth.label, + outcome.reason, + self._retry_seconds, + ) + return self._retry_seconds + verbose_proxy_logger.info( + "In-container pgbouncer picked up a fresh %s; the next one is due in %.0fs.", + self._source.auth.label, + outcome, + ) + return outcome + + def stop(self) -> None: + self._stopping.set() + if self._thread is not None: + self._thread.join() + + def _port_open(port: int) -> bool: try: with socket.create_connection((PGBOUNCER_LISTEN_ADDR, port), timeout=0.5): @@ -453,6 +610,11 @@ class PgBouncerProcess: ) threading.Thread(target=self._restart_after_delay, daemon=True, name="litellm-pgbouncer-supervisor").start() + def reload(self) -> None: + with self._lock: + if self._process is not None: + self._process.send_signal(signal.SIGHUP) + def stop(self) -> None: with self._lock: self._stopping.set() @@ -461,6 +623,39 @@ class PgBouncerProcess: _end(process) +def install_pgbouncer_token( + plan: PgBouncerPlan, runtime_dir: Path, run_as_user: str | None, pooler: PgBouncerProcess, token: str +) -> None: + write_userlist(plan.userlist(token), runtime_dir, run_as_user) + pooler.reload() + + +def _install_upstream_password( + plan: PgBouncerPlan, + runtime_dir: Path, + run_as_user: str | None, + pooler: PgBouncerProcess, + token_auth: DatabaseTokenAuth | None, + upstream_url: str, +) -> PgBouncerTokenRefresher | None | PgBouncerError: + if token_auth is None: + if plan.upstream_password is None: + return PgBouncerError( + "DATABASE_URL carries no password and neither IAM_TOKEN_DB_AUTH nor AZURE_POSTGRESQL_AUTH is on, " + "so the in-container PgBouncer has nothing to authenticate to Postgres with" + ) + write_userlist(plan.userlist(plan.upstream_password), runtime_dir, run_as_user) + return None + refresher: Final = PgBouncerTokenRefresher( + PgBouncerTokenSource(auth=token_auth, endpoint=parse_iam_endpoint_from_url(upstream_url)), + functools.partial(install_pgbouncer_token, plan, runtime_dir, run_as_user, pooler), + ) + failed: Final = refresher.start() + if failed is not None: + return failed + return refresher + + def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]: """An exit hook that does nothing in a forked child, which inherits the parent's ``atexit`` table.""" owner_pid: Final = os.getpid() @@ -475,7 +670,7 @@ def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]: def start_in_container_pgbouncer( settings: PgBouncerSettings, upstream_url: str, - token_auth_enabled: bool = False, + token_auth: DatabaseTokenAuth | None = None, register_exit_hook: Callable[[Callable[[], None]], object] = atexit.register, ) -> str | PgBouncerError: """Start the pooler for ``upstream_url`` and return the loopback URL the workers must use. @@ -484,10 +679,9 @@ def start_in_container_pgbouncer( once the worker manager has returned, and only by the process that started it (gunicorn forks its workers, so they carry the hooks too). PgBouncer refuses to run as root, so a root proxy (the default image) has it drop to - ``nobody``. + ``nobody``. With ``token_auth`` the password on ``upstream_url`` is ignored: + the pooler mints its own tokens and renews them for as long as it runs. """ - if token_auth_enabled: - return PgBouncerError(PGBOUNCER_TOKEN_AUTH_CONFLICT) version: Final = pgbouncer_version(settings.binary) if isinstance(version, PgBouncerError): return version @@ -503,7 +697,7 @@ def start_in_container_pgbouncer( plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user) if isinstance(plan, PgBouncerError): return plan - ini_path: Final = write_pgbouncer_files(plan, runtime_dir, run_as_user) + ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user) if isinstance(ini_path, PgBouncerError): return ini_path pooler: Final = PgBouncerProcess( @@ -511,15 +705,23 @@ def start_in_container_pgbouncer( port=settings.port, socket_path=unix_socket_path(runtime_dir, settings.port), ) + refresher: Final = _install_upstream_password(plan, runtime_dir, run_as_user, pooler, token_auth, upstream_url) + if isinstance(refresher, PgBouncerError): + return refresher failed: Final = pooler.start() if failed is not None: + if refresher is not None: + refresher.stop() return failed register_exit_hook(_only_in_this_process(pooler.stop)) + if refresher is not None: + register_exit_hook(_only_in_this_process(refresher.stop)) verbose_proxy_logger.info( - "In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections.", + "In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections%s.", pooler.pid, PGBOUNCER_LISTEN_ADDR, settings.port, settings.max_db_connections, + "" if token_auth is None else f" and renewing its {token_auth.label} before each one expires", ) return plan.pooled_url diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 16d76ff0415..c2d60cd5488 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -18,7 +18,12 @@ from pydantic import BaseModel, ConfigDict import litellm from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY -from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, start_in_container_pgbouncer +from litellm.proxy.db.pgbouncer import ( + PgBouncerError, + PgBouncerSettings, + export_pooled_database_url, + start_in_container_pgbouncer, +) from litellm.proxy.db.query_engine_reaper import start_query_engine_reaper if TYPE_CHECKING: @@ -1109,6 +1114,7 @@ def run_server( from litellm.proxy.db.token_auth import ( AZURE_POSTGRESQL_AUTH_ENV_VAR, IAM_TOKEN_DB_AUTH_ENV_VAR, + resolve_database_token_auth, token_auth_flag_enabled, ) @@ -1382,7 +1388,7 @@ def run_server( upstream_database_url: Final = os.getenv("DATABASE_URL") if pgbouncer_settings.enabled and upstream_database_url is not None: pooled_database_url: Final = start_in_container_pgbouncer( - pgbouncer_settings, upstream_database_url, token_auth_enabled=wants_rds_iam or wants_azure_entra + pgbouncer_settings, upstream_database_url, token_auth=resolve_database_token_auth() ) if isinstance(pooled_database_url, PgBouncerError): print( @@ -1392,7 +1398,7 @@ def run_server( flush=True, ) sys.exit(1) - os.environ["DATABASE_URL"] = pooled_database_url + export_pooled_database_url(pooled_database_url) if port == 4000 and ProxyInitializationHelpers._is_port_in_use(port): port = random.randint(1024, 49152) if prometheus_metrics_port == port: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 2cc58ca3110..253494b02f4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -131,6 +131,7 @@ from litellm.proxy.db.health_check_latest import ( fetch_latest_health_checks_for_models, ) from litellm.proxy.db.log_db_metrics import log_db_metrics +from litellm.proxy.db.pgbouncer import database_url_is_pooled from litellm.proxy.db.prisma_client import ( PrismaWrapper, parse_iam_endpoint_from_url, @@ -4007,6 +4008,7 @@ class PrismaClient: verbose_proxy_logger.error("Please run 'prisma generate' to generate the Prisma client.") raise Exception("Unable to find Prisma binaries. Please run 'prisma generate' first.") token_auth: Final = self.token_auth + writer_token_auth: Final = None if database_url_is_pooled() else token_auth # When read-replica routing is on, tag log lines with [writer]/[reader] # so the two wrappers' interleaved token refresh logs can be told apart. # Single-DB deployments get an empty prefix (logs unchanged). @@ -4015,13 +4017,13 @@ class PrismaClient: if http_client is not None: writer_wrapper = PrismaWrapper( original_prisma=Prisma(http=http_client), - token_auth=token_auth, + token_auth=writer_token_auth, log_prefix=writer_log_prefix, ) else: writer_wrapper = PrismaWrapper( original_prisma=Prisma(), - token_auth=token_auth, + token_auth=writer_token_auth, log_prefix=writer_log_prefix, ) diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 986428151e0..5ca45944c54 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -273,18 +273,17 @@ connections the pooler accepts. The module sets and the migration task keep the direct connection. ```hcl -create_database = false -database_url = "postgresql://litellm:@db.internal:5432/litellm" gateway_num_workers = 4 gateway_connection_pool_enabled = true gateway_pool_max_db_connections = 20 gateway_pool_max_client_conn = 1000 ``` -The pool needs a static database password, so it is only valid with an -existing database via `database_url`. The module-created Aurora authenticates -with rotating IAM tokens (see [Aurora + IAM auth](#aurora--iam-auth)), which -the pooler cannot follow, and `terraform plan` rejects that combination. +The pool works with the module-created Aurora as well as an existing database +via `database_url`. Against Aurora it authenticates with the same rotating IAM +tokens the workers used to (see [Aurora + IAM auth](#aurora--iam-auth)): the +pooler mints a token from the task role, renews it before it expires and hands +the workers a loopback URL with a static password instead The componentized `gateway_image` starts through `python -m gateway.launch`, which reads these variables, starts the pooler once per task and hands the diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 149842c14ff..7176921e8fa 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -306,8 +306,8 @@ resource "aws_ecs_task_definition" "gateway" { } precondition { - condition = !var.gateway_connection_pool_enabled || local.byo_database - error_message = "gateway_connection_pool_enabled requires an existing database via database_url with create_database = false: the module-created Aurora authenticates with IAM tokens, which the in-container pgbouncer cannot follow because it holds a static database password." + condition = !var.gateway_connection_pool_enabled || local.database_enabled + error_message = "gateway_connection_pool_enabled needs a database: set create_database = true or pass database_url." } } diff --git a/terraform/litellm/aws/tests/connection_pool.tftest.hcl b/terraform/litellm/aws/tests/connection_pool.tftest.hcl index 38aa7051134..ab0a1cfc076 100644 --- a/terraform/litellm/aws/tests/connection_pool.tftest.hcl +++ b/terraform/litellm/aws/tests/connection_pool.tftest.hcl @@ -88,16 +88,20 @@ run "gateway_starts_through_the_pool_aware_launcher" { } } -run "pool_with_module_created_iam_aurora_fails_at_plan" { +run "pool_with_module_created_iam_aurora_plans_with_both_the_pool_and_iam_auth" { command = plan variables { gateway_connection_pool_enabled = true } - expect_failures = [ - aws_ecs_task_definition.gateway, - ] + assert { + condition = alltrue([ + length(local.gateway_pool_env) == 3, + contains(local.managed_db_env, { name = "IAM_TOKEN_DB_AUTH", value = "true" }), + ]) + error_message = "With the module-created Aurora the gateway must get the pool env alongside IAM token auth." + } } run "pool_without_any_database_fails_at_plan" { diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index ec8363dbbaf..f57bb1ab7f6 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -208,10 +208,9 @@ variable "gateway_connection_pool_enabled" { Postgres, so a task's footprint against the database connection ceiling is workers x connection_limit and grows with every task. Sets LITELLM_PGBOUNCER_ENABLED / LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / - LITELLM_PGBOUNCER_MAX_CLIENT_CONN on the gateway container only. Requires - an existing database via `database_url`: the module-created Aurora - authenticates with IAM tokens, which the pooler cannot follow because it - holds one static password for the life of the task. + LITELLM_PGBOUNCER_MAX_CLIENT_CONN on the gateway container only. Works with + the module-created Aurora too: the pooler mints the IAM token itself and + renews it before it expires. EOT type = bool default = false diff --git a/tests/test_gateway/test_launch.py b/tests/test_gateway/test_launch.py index 964a7021416..6039aaafc2b 100644 --- a/tests/test_gateway/test_launch.py +++ b/tests/test_gateway/test_launch.py @@ -5,6 +5,7 @@ import textwrap import urllib.parse from pathlib import Path from typing import Final, cast +from unittest.mock import MagicMock, patch import pytest from uvicorn.importer import import_from_string @@ -13,7 +14,7 @@ from uvicorn.main import main as uvicorn_main import gateway.main from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv from litellm.proxy.db.db_url_settings import DatabaseURLSettings -from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings +from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -66,7 +67,13 @@ def _query(url: str) -> dict[str, str]: @pytest.fixture def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: - for var in ("DATABASE_URL", "IAM_TOKEN_DB_AUTH", "AZURE_POSTGRESQL_AUTH", "DATABASE_HOST_READ_REPLICA"): + for var in ( + "DATABASE_URL", + "IAM_TOKEN_DB_AUTH", + "AZURE_POSTGRESQL_AUTH", + "DATABASE_HOST_READ_REPLICA", + PGBOUNCER_POOLED_ENV_VAR, + ): monkeypatch.setenv(var, "") monkeypatch.delenv(var) for var, value in DB_ENV.items(): @@ -74,6 +81,12 @@ def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: return dict(DB_ENV) +def _minted_iam_token(token: str): + rds: Final = MagicMock() + rds.generate_db_auth_token.return_value = token + return patch("boto3.client", return_value=rds) + + def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: return uvicorn_main.make_context("uvicorn", list(argv)).params @@ -109,16 +122,23 @@ class TestPoolDatabaseUrl: assert isinstance(outcome, PgBouncerError) assert "DATABASE_URL" in outcome.reason - def test_token_auth_is_refused(self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + def test_token_auth_hands_the_workers_the_pool_user_not_the_token( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") - environ: Final = {"DATABASE_URL": "postgresql://litellm:token@db.internal:5432/litellm"} - outcome: Final = pool_database_url( - DatabaseURLSettings.from_env(), - PgBouncerSettings(enabled=True, port=_free_port(), binary=str(_fake_pooler(tmp_path))), - environ, - ) - assert isinstance(outcome, PgBouncerError) - assert "IAM_TOKEN_DB_AUTH" in outcome.reason + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + port: Final = _free_port() + environ: Final = {"DATABASE_URL": "postgresql://litellm:MINTED_TOKEN@db.internal:5432/litellm"} + with _minted_iam_token("MINTED_TOKEN"): + outcome: Final = pool_database_url( + DatabaseURLSettings.from_env(), + PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path))), + environ, + ) + assert isinstance(outcome, str), outcome + pooled: Final = urllib.parse.urlsplit(outcome) + assert (pooled.username, pooled.hostname, pooled.port) == ("litellm_pgbouncer", "127.0.0.1", port) + assert "MINTED_TOKEN" not in outcome class TestMain: @@ -134,14 +154,39 @@ class TestMain: main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) pooled: Final = os.environ["DATABASE_URL"] - assert urllib.parse.urlsplit(pooled).netloc == f"litellm_pool:p%40ss@127.0.0.1:{port}" + assert urllib.parse.urlsplit(pooled).hostname == "127.0.0.1" + assert urllib.parse.urlsplit(pooled).port == port + assert urllib.parse.urlsplit(pooled).username == "litellm_pgbouncer" + assert "p%40ss" not in pooled assert _query(pooled)["pgbouncer"] == "true" assert _uvicorn_params(served[0])["timeout_keep_alive"] == 75 DatabaseURLSettings.from_env().apply_to_env() - worker_url: Final = os.environ["DATABASE_URL"] - assert urllib.parse.urlsplit(worker_url).netloc == f"litellm_pool:p%40ss@127.0.0.1:{port}" - assert _query(worker_url)["pgbouncer"] == "true" + assert urllib.parse.urlsplit(os.environ["DATABASE_URL"]).netloc == urllib.parse.urlsplit(pooled).netloc + assert _query(os.environ["DATABASE_URL"])["pgbouncer"] == "true" + + def test_iam_workers_keep_the_loopback_url_instead_of_minting_their_own( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): + port: Final = _free_port() + monkeypatch.delenv("DATABASE_PASSWORD") + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + monkeypatch.setenv("LITELLM_PGBOUNCER_ENABLED", "true") + monkeypatch.setenv("LITELLM_PGBOUNCER_PORT", str(port)) + monkeypatch.setenv("LITELLM_PGBOUNCER_BINARY", str(_fake_pooler(tmp_path))) + served: Final[list[tuple[str, ...]]] = [] + with _minted_iam_token("SUPERVISOR_TOKEN"): + main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) + pooled: Final = os.environ["DATABASE_URL"] + assert urllib.parse.urlsplit(pooled).netloc.endswith(f"@127.0.0.1:{port}") + assert "SUPERVISOR_TOKEN" not in pooled + assert os.environ[PGBOUNCER_POOLED_ENV_VAR] == "true" + assert len(served) == 1 + + with _minted_iam_token("WORKER_TOKEN"): + DatabaseURLSettings.from_env().apply_to_env() + assert os.environ["DATABASE_URL"] == pooled def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path diff --git a/tests/test_litellm/proxy/db/test_pgbouncer.py b/tests/test_litellm/proxy/db/test_pgbouncer.py index da1d4c91f09..7b5a0bf10f9 100644 --- a/tests/test_litellm/proxy/db/test_pgbouncer.py +++ b/tests/test_litellm/proxy/db/test_pgbouncer.py @@ -1,4 +1,6 @@ +import base64 import configparser +import json import logging import os import signal @@ -9,8 +11,10 @@ import tempfile import textwrap import time import urllib.parse +from collections import deque from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final, cast @@ -18,16 +22,24 @@ import pytest from litellm._logging import verbose_proxy_logger from litellm.proxy.db.pgbouncer import ( + PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerPlan, PgBouncerProcess, PgBouncerSettings, + PgBouncerTokenRefresher, + PgBouncerTokenSource, + database_url_is_pooled, + export_pooled_database_url, + install_pgbouncer_token, pgbouncer_version, plan_pgbouncer, start_in_container_pgbouncer, unix_socket_path, - write_pgbouncer_files, + write_pgbouncer_ini, + write_userlist, ) +from litellm.proxy.db.token_auth import AzureEntraTokenAuth, IAMEndpoint UPSTREAM: Final = ( "postgresql://app:p%40ss%27w@db.internal:5433/litellm" @@ -55,17 +67,24 @@ def _query(url: str) -> dict[str, str]: class TestPlanPgBouncer: - def test_upstream_credentials_and_timeouts_move_into_the_pgbouncer_config(self): + def test_upstream_route_and_timeouts_move_into_the_pgbouncer_config_without_the_password(self): ini: Final = _ini(_plan()) assert ini["databases"]["litellm"] == ( - "host='db.internal' port=5433 dbname='litellm' user='app' password='p@ss''w' " + "host='db.internal' port=5433 dbname='litellm' user='app' " "connect_query='SET statement_timeout TO ''7000''; SET lock_timeout TO ''3000'''" ) - assert _plan().userlist == '"app" "p@ss\'w"\n' + + def test_the_auth_file_holds_the_upstream_password_and_the_pool_users_own(self): + plan: Final = _plan() + assert plan.upstream_password == "p@ss'w" + assert plan.userlist("p@ss'w") == f'"app" "p@ss\'w"\n"litellm_pgbouncer" "{plan.pool_password}"\n' + + def test_a_token_with_quotes_is_escaped_the_way_pgbouncer_reads_it(self): + assert _plan().userlist('to"ken').startswith('"app" "to""ken"\n') def test_an_upstream_without_a_port_is_reached_on_the_postgres_default(self): ini: Final = _ini(_plan("postgresql://app:pw@db/litellm")) - assert ini["databases"]["litellm"] == "host='db' port=5432 dbname='litellm' user='app' password='pw'" + assert ini["databases"]["litellm"] == "host='db' port=5432 dbname='litellm' user='app'" def test_pool_is_sized_from_settings_in_transaction_mode(self): pgb: Final = _ini(_plan())["pgbouncer"] @@ -79,20 +98,24 @@ class TestPlanPgBouncer: assert pgb["auth_file"] == "/run/pgb/userlist.txt" assert pgb["unix_socket_dir"] == "/run/pgb" - def test_the_app_user_can_read_the_pgbouncer_console(self): - assert _ini(_plan())["pgbouncer"]["stats_users"] == "app" + def test_the_pool_user_can_read_the_pgbouncer_console(self): + assert _ini(_plan())["pgbouncer"]["stats_users"] == "litellm_pgbouncer" - @pytest.mark.parametrize("user", ["app,admin", "app%20admin", "app%09admin"]) - def test_a_user_pgbouncer_would_split_into_several_console_users_is_refused(self, user: str): - outcome: Final = plan_pgbouncer(f"postgresql://{user}:pw@db/litellm", SETTINGS, Path("/run/pgb"), None) + def test_a_database_user_named_like_the_pool_user_is_refused(self): + outcome: Final = plan_pgbouncer( + "postgresql://litellm_pgbouncer:pw@db/litellm", SETTINGS, Path("/run/pgb"), None + ) assert isinstance(outcome, PgBouncerError) - assert "stats_users" in outcome.reason + assert "litellm_pgbouncer" in outcome.reason - def test_pooled_url_points_prisma_at_loopback_without_prepared_statements(self): - pooled: Final = urllib.parse.urlsplit(_plan().pooled_url) + def test_pooled_url_points_prisma_at_loopback_as_the_pool_user_without_prepared_statements(self): + plan: Final = _plan() + pooled: Final = urllib.parse.urlsplit(plan.pooled_url) assert (pooled.hostname, pooled.port, pooled.path) == ("127.0.0.1", 6543, "/litellm") - assert (pooled.username, pooled.password) == ("app", "p%40ss%27w") - assert _query(_plan().pooled_url) == { + assert (pooled.username, pooled.password) == ("litellm_pgbouncer", plan.pool_password) + assert len(plan.pool_password) >= 32 + assert "p%40ss" not in plan.pooled_url + assert _query(plan.pooled_url) == { "schema": "public", "connection_limit": "10", "pool_timeout": "20", @@ -118,6 +141,9 @@ class TestPlanPgBouncer: assert "server_tls_ca_file" not in pgb assert plan.ca_source is None + def test_every_plan_gets_its_own_pool_password(self): + assert _plan().pool_password != _plan().pool_password + def test_no_tls_params_default_to_prefer(self): assert _ini(_plan("postgresql://app:pw@db/litellm"))["pgbouncer"]["server_tls_sslmode"] == "prefer" @@ -138,15 +164,20 @@ class TestPlanPgBouncer: @pytest.mark.parametrize( "url", [ - "postgresql://app@db/litellm", "postgresql://app:pw@db", "postgresql://:pw@db/litellm", + "postgresql://app:pw@/litellm", ], ) - def test_urls_missing_forwardable_credentials_are_refused(self, url: str): + def test_urls_missing_a_route_are_refused(self, url: str): outcome: Final = plan_pgbouncer(url, SETTINGS, Path("/run/pgb"), None) assert isinstance(outcome, PgBouncerError) + def test_a_url_without_a_password_plans_for_a_token_to_be_installed_later(self): + plan: Final = _plan("postgresql://app@db/litellm") + assert plan.upstream_password is None + assert plan.userlist("minted").startswith('"app" "minted"\n') + def test_every_options_spelling_becomes_a_set_statement(self): options: Final = urllib.parse.quote("-c a=1 -cb=2 --c=3") ini: Final = _ini(_plan(f"postgresql://app:pw@db/litellm?options={options}")) @@ -167,12 +198,14 @@ class TestPlanPgBouncer: class TestWritePgBouncerFiles: def test_files_hold_the_plan_and_are_private_to_the_owner(self, tmp_path: Path): plan: Final = _plan("postgresql://app:pw@db/litellm") - ini_path: Final = write_pgbouncer_files(plan, tmp_path, None) + ini_path: Final = write_pgbouncer_ini(plan, tmp_path, None) assert isinstance(ini_path, Path), ini_path + userlist_path: Final = write_userlist(plan.userlist("pw"), tmp_path, None) assert ini_path == tmp_path / "pgbouncer.ini" + assert userlist_path == tmp_path / "userlist.txt" assert ini_path.read_text() == plan.ini - assert (tmp_path / "userlist.txt").read_text() == plan.userlist - for path in (ini_path, tmp_path / "userlist.txt"): + assert userlist_path.read_text() == plan.userlist("pw") + for path in (ini_path, userlist_path): assert stat.S_IMODE(path.stat().st_mode) == 0o600 assert not (tmp_path / "server-ca.pem").exists() @@ -185,7 +218,7 @@ class TestWritePgBouncerFiles: f"postgresql://app:pw@db/litellm?sslmode=verify-full&sslcert={bundle}", SETTINGS, runtime_dir, None ) assert isinstance(plan, PgBouncerPlan), plan - ini_path: Final = write_pgbouncer_files(plan, runtime_dir, None) + ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, None) assert isinstance(ini_path, Path), ini_path ca_file: Final = Path(_ini(plan)["pgbouncer"]["server_tls_ca_file"]) assert ca_file.parent == runtime_dir @@ -199,11 +232,32 @@ class TestWritePgBouncerFiles: None, ) assert isinstance(plan, PgBouncerPlan), plan - outcome: Final = write_pgbouncer_files(plan, tmp_path, None) + outcome: Final = write_pgbouncer_ini(plan, tmp_path, None) assert isinstance(outcome, PgBouncerError) assert "missing.pem" in outcome.reason assert not (tmp_path / "pgbouncer.ini").exists() + def test_rewriting_the_userlist_replaces_it_whole_and_leaves_nothing_else_behind(self, tmp_path: Path): + write_userlist('"app" "first"\n', tmp_path, None) + with open(tmp_path / "userlist.txt", encoding="utf-8") as before_rewrite: + write_userlist('"app" "second"\n', tmp_path, None) + assert before_rewrite.read() == '"app" "first"\n' + assert (tmp_path / "userlist.txt").read_text() == '"app" "second"\n' + assert stat.S_IMODE((tmp_path / "userlist.txt").stat().st_mode) == 0o600 + assert sorted(path.name for path in tmp_path.iterdir()) == ["userlist.txt"] + + +class TestPooledUrlMarker: + def test_exporting_the_pooled_url_marks_it_for_the_workers(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(PGBOUNCER_POOLED_ENV_VAR, "") + monkeypatch.delenv(PGBOUNCER_POOLED_ENV_VAR) + monkeypatch.setenv("DATABASE_URL", "postgresql://app:token@db/litellm") + assert not database_url_is_pooled() + export_pooled_database_url("postgresql://litellm_pgbouncer:pw@127.0.0.1:6432/litellm?pgbouncer=true") + assert os.environ["DATABASE_URL"] == "postgresql://litellm_pgbouncer:pw@127.0.0.1:6432/litellm?pgbouncer=true" + assert database_url_is_pooled() + assert PGBOUNCER_POOLED_ENV_VAR == "LITELLM_PGBOUNCER_POOLED_DATABASE_URL" + def _bound_port(sock: socket.socket) -> int: return cast(tuple[str, int], sock.getsockname())[1] @@ -222,20 +276,23 @@ def _fake_pooler( port_file: Path | None = None, bind_delay_seconds: float = 0.0, version_banner: str = "PgBouncer 1.25.2\nlibevent 2.1.13-stable", + auth_log: Path | None = None, ) -> Path: """An executable that listens like PgBouncer: on the TCP port first, then on ``.s.PGSQL.`` in the socket dir. Port and socket dir come from the ini it is given, else from ``port`` and ``tmp_path``. With ``port_file`` each start reads the port from that file instead. ``bind_delay_seconds`` holds the bind back, like a slow start. - ``--version`` prints ``version_banner``. + ``--version`` prints ``version_banner``. With ``auth_log`` it appends the + ``auth_file`` it reads at startup and on every SIGHUP, one line per read, + like PgBouncer loading its credentials. """ script: Final = tmp_path / "fake-pgbouncer" script.write_text( textwrap.dedent( f"""\ #!{sys.executable} - import configparser, os, pathlib, select, socket, sys, time + import configparser, os, pathlib, select, signal, socket, sys, time if sys.argv[1:] == ["--version"]: print({version_banner!r}) sys.exit(0) @@ -243,6 +300,12 @@ def _fake_pooler( sys.exit(3) ini = configparser.ConfigParser() ini.read(sys.argv[1:2]) + if not {auth_log is None!r}: + def load_auth_file(*_): + with open({str(auth_log)!r}, "a") as log: + log.write(repr(pathlib.Path(ini.get("pgbouncer", "auth_file")).read_text()) + "\\n") + load_auth_file() + signal.signal(signal.SIGHUP, load_auth_file) port = ini.getint("pgbouncer", "listen_port", fallback={port}) if not {port_file is None!r}: port = int(pathlib.Path({str(port_file)!r}).read_text()) @@ -511,6 +574,118 @@ class TestPgBouncerProcess: os.kill(pid, 0) +NOW: Final = datetime(2026, 9, 10, 12, 0, tzinfo=timezone.utc) +ENDPOINT: Final = IAMEndpoint(host="db", port="5432", user="app", name="litellm") + + +def _entra_jwt(expires_at: datetime) -> str: + payload: Final = base64.urlsafe_b64encode(json.dumps({"exp": int(expires_at.timestamp())}).encode()) + return f"aGVhZGVy.{payload.rstrip(b'=').decode()}.c2ln" + + +def _token_source(*tokens: str | Exception) -> PgBouncerTokenSource: + """A token source handing out ``tokens`` in order, raising the exceptions among them, then repeating the last.""" + pending: Final = deque(tokens) + + def provide() -> str: + outcome: Final = pending.popleft() if len(pending) > 1 else pending[0] + if isinstance(outcome, Exception): + raise outcome + return outcome + + return PgBouncerTokenSource(auth=AzureEntraTokenAuth(token_provider=provide), endpoint=ENDPOINT) + + +class TestPgBouncerTokenRefresher: + def _refresher( + self, + source: PgBouncerTokenSource, + installed: list[str], + install: Callable[[str], None] | None = None, + **timing: float, + ) -> PgBouncerTokenRefresher: + return PgBouncerTokenRefresher( + source, + install if install is not None else installed.append, + now=lambda: NOW.replace(tzinfo=None), + **timing, + ) + + def test_the_next_refresh_is_due_a_buffer_before_the_token_expires(self): + installed: Final[list[str]] = [] + token: Final = _entra_jwt(NOW + timedelta(hours=1)) + refresher: Final = self._refresher(_token_source(token), installed, buffer_seconds=180) + assert refresher.refresh() == 3600 - 180 + assert installed == [token] + + def test_a_token_whose_expiry_cannot_be_read_is_refreshed_on_the_fallback_interval(self): + installed: Final[list[str]] = [] + refresher: Final = self._refresher(_token_source("opaque token"), installed, fallback_seconds=600) + assert refresher.refresh() == 600 + assert installed == ["opaque token"] + + def test_a_token_already_inside_the_buffer_is_refreshed_after_the_retry_delay(self): + token: Final = _entra_jwt(NOW + timedelta(seconds=100)) + refresher: Final = self._refresher(_token_source(token), [], buffer_seconds=180, retry_seconds=30) + assert refresher.refresh() == 30 + + def test_the_token_reaches_the_auth_file_in_wire_form_not_url_encoded(self): + installed: Final[list[str]] = [] + self._refresher(_token_source("to ken/with+odd=chars"), installed).refresh() + assert installed == ["to ken/with+odd=chars"] + + def test_a_failed_mint_is_reported_and_installs_nothing(self): + installed: Final[list[str]] = [] + outcome: Final = self._refresher(_token_source(RuntimeError("no credential")), installed).refresh() + assert isinstance(outcome, PgBouncerError) + assert "Azure Entra token" in outcome.reason + assert "no credential" in outcome.reason + assert installed == [] + + def test_a_token_pgbouncer_cannot_hold_is_refused(self): + installed: Final[list[str]] = [] + outcome: Final = self._refresher(_token_source("x" * 2048), installed).refresh() + assert isinstance(outcome, PgBouncerError) + assert "2047" in outcome.reason + assert installed == [] + + def test_an_auth_file_that_cannot_be_written_is_reported_not_raised(self): + def refuse(_: str) -> None: + raise PermissionError("read-only runtime dir") + + outcome: Final = self._refresher(_token_source("token"), [], install=refuse).refresh() + assert isinstance(outcome, PgBouncerError) + assert "read-only runtime dir" in outcome.reason + + def test_start_fails_when_the_first_token_cannot_be_minted_and_schedules_nothing(self): + installed: Final[list[str]] = [] + refresher: Final = self._refresher( + _token_source(RuntimeError("no credential"), "later"), installed, fallback_seconds=0.05 + ) + assert isinstance(refresher.start(), PgBouncerError) + time.sleep(0.3) + assert installed == [] + + def test_a_failed_renewal_keeps_the_previous_token_until_the_retry_succeeds(self, caplog: pytest.LogCaptureFixture): + installed: Final[list[str]] = [] + refresher: Final = self._refresher( + _token_source("first", RuntimeError("blip"), "third"), + installed, + fallback_seconds=0.05, + retry_seconds=0.05, + ) + with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + assert refresher.start() is None + assert installed == ["first"] + assert _wait_until(lambda: "third" in installed) + assert installed[:2] == ["first", "third"] + assert any("keeps its current Azure Entra token" in record.message for record in caplog.records) + refresher.stop() + settled: Final = len(installed) + time.sleep(0.3) + assert len(installed) == settled + + def _runtime_dir_listening_on(port: int) -> Path: matches: Final = tuple( ini.parent @@ -524,10 +699,21 @@ def _runtime_dir_listening_on(port: int) -> Path: class TestStartInContainerPgBouncer: def test_returns_the_loopback_url_once_the_pooler_listens(self, tmp_path: Path): port: Final = _free_port() - settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + auth_log: Final = tmp_path / "auth.log" + binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log) + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm?connection_limit=5") - assert pooled == f"postgresql://app:pw@127.0.0.1:{port}/litellm?connection_limit=5&pgbouncer=true" + assert isinstance(pooled, str), pooled + parsed: Final = urllib.parse.urlsplit(pooled) + assert (parsed.username, parsed.hostname, parsed.port, parsed.path) == ( + "litellm_pgbouncer", + "127.0.0.1", + port, + "/litellm", + ) + assert _query(pooled) == {"connection_limit": "5", "pgbouncer": "true"} assert _listening(port) + assert auth_log.read_text() == repr(f'"app" "pw"\n"litellm_pgbouncer" "{parsed.password}"\n') + "\n" @pytest.mark.filterwarnings("ignore:This process .* is multi-threaded:DeprecationWarning") def test_a_forked_worker_exiting_leaves_the_pooler_and_its_files_to_the_parent(self, tmp_path: Path): @@ -561,21 +747,79 @@ class TestStartInContainerPgBouncer: def test_a_bad_upstream_url_is_reported_without_starting_anything(self, tmp_path: Path): port: Final = _free_port() settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) - outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app@db/litellm") + outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db") assert isinstance(outcome, PgBouncerError) assert not _listening(port) - def test_token_auth_is_refused_without_starting_anything(self, tmp_path: Path): + def test_a_passwordless_url_without_token_auth_is_refused_without_starting_anything(self, tmp_path: Path): port: Final = _free_port() settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) - outcome: Final = start_in_container_pgbouncer( - settings, "postgresql://app:pw@db/litellm", token_auth_enabled=True - ) + outcome: Final = start_in_container_pgbouncer(settings, "postgresql://app@db/litellm") assert isinstance(outcome, PgBouncerError) assert "IAM_TOKEN_DB_AUTH" in outcome.reason - assert "AZURE_POSTGRESQL_AUTH" in outcome.reason assert not _listening(port) + def test_token_auth_mints_the_first_token_into_the_auth_file_before_the_pooler_starts(self, tmp_path: Path): + port: Final = _free_port() + auth_log: Final = tmp_path / "auth.log" + binary: Final = _fake_pooler(tmp_path, port, auth_log=auth_log) + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) + token: Final = _entra_jwt(datetime.now(tz=timezone.utc) + timedelta(hours=1)) + pooled: Final = start_in_container_pgbouncer( + settings, + "postgresql://app:stale-token@db/litellm", + token_auth=AzureEntraTokenAuth(token_provider=lambda: token), + ) + assert isinstance(pooled, str), pooled + parsed: Final = urllib.parse.urlsplit(pooled) + assert parsed.username == "litellm_pgbouncer" + assert token not in pooled + assert _listening(port) + assert auth_log.read_text() == repr(f'"app" "{token}"\n"litellm_pgbouncer" "{parsed.password}"\n') + "\n" + + def test_a_first_token_that_cannot_be_minted_is_reported_without_starting_anything(self, tmp_path: Path): + port: Final = _free_port() + settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(_fake_pooler(tmp_path, port))) + + def fail() -> str: + raise RuntimeError("no Azure credential") + + outcome: Final = start_in_container_pgbouncer( + settings, "postgresql://app@db/litellm", token_auth=AzureEntraTokenAuth(token_provider=fail) + ) + assert isinstance(outcome, PgBouncerError) + assert "no Azure credential" in outcome.reason + assert not _listening(port) + + def test_a_renewed_token_is_written_and_picked_up_by_the_running_and_by_a_restarted_pooler(self, tmp_path: Path): + port: Final = _free_port() + auth_log: Final = tmp_path / "auth.log" + plan: Final = plan_pgbouncer( + "postgresql://app@db/litellm", PgBouncerSettings(enabled=True, port=port), tmp_path, None + ) + assert isinstance(plan, PgBouncerPlan), plan + ini_path: Final = write_pgbouncer_ini(plan, tmp_path, None) + write_userlist(plan.userlist("first"), tmp_path, None) + pooler: Final = PgBouncerProcess( + argv=(str(_fake_pooler(tmp_path, port, auth_log=auth_log)), str(ini_path)), + port=port, + socket_path=unix_socket_path(tmp_path, port), + restart_delay_seconds=0.1, + ) + assert pooler.start() is None + first_pid: Final = pooler.pid + assert first_pid is not None + install_pgbouncer_token(plan, tmp_path, None, pooler, "second") + assert _wait_until(lambda: auth_log.read_text().count("\n") == 2) + os.kill(first_pid, signal.SIGKILL) + assert _wait_until(lambda: pooler.pid not in (None, first_pid) and _listening(port)) + pooler.stop() + assert auth_log.read_text().splitlines() == [ + repr(plan.userlist("first")), + repr(plan.userlist("second")), + repr(plan.userlist("second")), + ] + def test_a_pgbouncer_that_survives_a_failed_tcp_bind_is_refused_without_starting(self, tmp_path: Path): port: Final = _free_port() binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.18.1\nlibevent 2.1.12-stable") @@ -590,9 +834,9 @@ class TestStartInContainerPgBouncer: port: Final = _free_port() binary: Final = _fake_pooler(tmp_path, port, version_banner="PgBouncer 1.19.0") settings: Final = PgBouncerSettings(enabled=True, port=port, binary=str(binary)) - assert start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") == ( - f"postgresql://app:pw@127.0.0.1:{port}/litellm?pgbouncer=true" - ) + pooled: Final = start_in_container_pgbouncer(settings, "postgresql://app:pw@db/litellm") + assert isinstance(pooled, str), pooled + assert urllib.parse.urlsplit(pooled).port == port assert _listening(port) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 1287cf23157..646596c5b88 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -1,5 +1,6 @@ import datetime as real_datetime import smtplib +from typing import Final import pytest from fastapi import HTTPException @@ -8,7 +9,7 @@ from litellm.caching.caching import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.types.guardrails import GuardrailEventHooks @@ -2207,3 +2208,49 @@ async def test_post_call_failure_hook_redacts_traceback_before_callbacks(monkeyp assert recorder.received_traceback is not None assert provider_key not in recorder.received_traceback assert "REDACTED" in recorder.received_traceback + + +class TestPrismaClientTokenAuthBehindThePool: + """Behind the in-container pool the supervisor renews the writer's database + token and hands the workers a loopback URL with a static password, so the + writer wrapper must not run its own refresh loop. The reader is not pooled + and keeps refreshing its own token.""" + + UPSTREAM: Final = "postgresql://litellm:TOKEN@db.internal:5432/litellm" + READER: Final = "postgresql://litellm:TOKEN@reader.internal:5432/litellm" + + def _client(self, monkeypatch: pytest.MonkeyPatch, pooled: bool) -> PrismaClient: + from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR + + monkeypatch.delenv("AZURE_POSTGRESQL_AUTH", raising=False) + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + monkeypatch.setenv("DATABASE_URL", self.UPSTREAM) + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", self.READER) + if pooled: + monkeypatch.setenv(PGBOUNCER_POOLED_ENV_VAR, "true") + else: + monkeypatch.delenv(PGBOUNCER_POOLED_ENV_VAR, raising=False) + rds: Final = MagicMock() + rds.generate_db_auth_token.return_value = "TOKEN" + with patch("boto3.client", return_value=rds): + return PrismaClient(database_url=self.UPSTREAM, proxy_logging_obj=MagicMock(spec=ProxyLogging)) + + def test_a_pooled_writer_leaves_token_refresh_to_the_pooler_while_the_reader_keeps_its_own( + self, monkeypatch: pytest.MonkeyPatch + ): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + client = self._client(monkeypatch, pooled=True) + assert isinstance(client.db, RoutingPrismaWrapper) + assert client.db.writer.iam_token_db_auth is False + assert client.db.reader.iam_token_db_auth is True + assert client.token_auth is not None + + def test_an_unpooled_writer_still_refreshes_its_own_token(self, monkeypatch: pytest.MonkeyPatch): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + client = self._client(monkeypatch, pooled=False) + assert isinstance(client.db, RoutingPrismaWrapper) + assert client.db.writer.iam_token_db_auth is True + assert client.db.reader.iam_token_db_auth is True From 033f2e5e2a651fed59faf6478fb18314e487738b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 14:19:08 -0700 Subject: [PATCH 382/411] feat(wandb): default unmapped W&B models to reasoning-capable W&B's serverless catalog grows faster than the registry names it, so a model they ship today resolves as non-reasoning here until someone edits the cost map, and the caller's reasoning_effort is dropped or rejected. Add a wandb-reasoning-baseline capability rule to fallback_generalizations so any wandb/ id the map has not described defaults to supports_reasoning. Rules lose to exact entries, so mapped non-reasoning models such as wandb/meta-llama/Llama-3.1-8B-Instruct are unaffected. The rule carries no mode and no pricing, so cost stays on the standard unpriced behavior and the deployment does not read as catalog-mapped to the router's reasoning-effort resolver. Claude-Session: https://claude.ai/code/session_01A6SkwJdfZUmkzfUkrEkqX8 --- ...odel_prices_and_context_window_backup.json | 8 +++ model_prices_and_context_window.json | 8 +++ .../test_fallback_generalizations.py | 60 +++++++++++++++++++ .../wandb/test_wandb_chat_transformation.py | 26 +++++++- 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1c0bd32d782..b7726290f0e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57258,6 +57258,14 @@ "model_info": { "supports_mid_conversation_system": true } + }, + { + "name": "wandb-reasoning-baseline", + "pattern": "^wandb/", + "description": "Any Weights & Biases Inference model id, anchored to the wandb/ namespace so only that provider's ids match. W&B's serverless catalog is reasoning-first and grows faster than this registry names it, so an id the map has not described yet is treated as reasoning-capable and keeps the caller's reasoning_effort instead of dropping it or raising UnsupportedParamsError. Rules lose to exact entries, so a mapped non-reasoning model such as wandb/meta-llama/Llama-3.1-8B-Instruct is unaffected. Carries no mode and no pricing, so cost stays on the standard unpriced behavior and the deployment does not read as catalog-mapped to the router's reasoning-effort resolver.", + "model_info": { + "supports_reasoning": true + } } ] }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1c0bd32d782..b7726290f0e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57258,6 +57258,14 @@ "model_info": { "supports_mid_conversation_system": true } + }, + { + "name": "wandb-reasoning-baseline", + "pattern": "^wandb/", + "description": "Any Weights & Biases Inference model id, anchored to the wandb/ namespace so only that provider's ids match. W&B's serverless catalog is reasoning-first and grows faster than this registry names it, so an id the map has not described yet is treated as reasoning-capable and keeps the caller's reasoning_effort instead of dropping it or raising UnsupportedParamsError. Rules lose to exact entries, so a mapped non-reasoning model such as wandb/meta-llama/Llama-3.1-8B-Instruct is unaffected. Carries no mode and no pricing, so cost stays on the standard unpriced behavior and the deployment does not read as catalog-mapped to the router's reasoning-effort resolver.", + "model_info": { + "supports_reasoning": true + } } ] }, diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index b1e8163b91d..56552ec20ab 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -571,3 +571,63 @@ def test_shipped_mid_conversation_gate_on_bedrock_ids(shipped_cost_map): ): matched = match_capability_generalizations(unflagged) assert matched is None or not matched.get("supports_mid_conversation_system"), unflagged + + +def test_shipped_rules_flag_unmapped_wandb_ids_as_reasoning(shipped_cost_map): + """W&B ships reasoning models faster than the registry names them, so an unmapped + wandb id resolves as reasoning-capable and its reasoning_effort survives instead of + being dropped. The rule carries no mode and no pricing, so cost stays on the standard + unpriced behavior and the deployment does not read as catalog-mapped.""" + model = "wandb/zai-org/GLM-6-Turbo" + assert model not in litellm.model_cost + + info = litellm.get_model_info(model, custom_llm_provider="wandb") + assert info["litellm_provider"] == "wandb" + assert info["supports_reasoning"] is True + assert info.get("mode") is None + assert not info.get("input_cost_per_token") + assert not info.get("output_cost_per_token") + + assert litellm.supports_reasoning(model="zai-org/GLM-6-Turbo", custom_llm_provider="wandb") is True + + +def test_shipped_wandb_rule_loses_to_mapped_non_reasoning_entries(shipped_cost_map): + """The whole point of a fallback is that it only fills gaps. A wandb model the map + describes as non-reasoning must stay non-reasoning, otherwise the rule silently + re-introduces the blanket supports_reasoning it exists to avoid.""" + for model in ( + "meta-llama/Llama-3.1-8B-Instruct", + "microsoft/Phi-4-mini-instruct", + "moonshotai/Kimi-K2-Instruct", + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + ): + assert f"wandb/{model}" in litellm.model_cost, model + assert litellm.supports_reasoning(model=model, custom_llm_provider="wandb") is False, model + + +def test_shipped_wandb_rule_is_anchored_to_the_wandb_namespace(shipped_cost_map): + """``^wandb/`` is anchored, so it cannot leak onto another provider's ids.""" + assert match_capability_generalizations("wandb/some-new-model") == {"supports_reasoning": True} + for foreign in ("openai/some-new-model", "notwandb/some-new-model", "together_ai/wandb/some-new-model"): + matched = match_capability_generalizations(foreign) + assert matched is None or not matched.get("supports_reasoning"), foreign + + +def test_shipped_wandb_rule_keeps_reasoning_effort_on_an_unmapped_model(shipped_cost_map): + """End to end through the provider config: the gate WandbConfig applies reads the + rule, so reasoning_effort is advertised and survives get_optional_params rather than + raising UnsupportedParamsError.""" + model = "zai-org/GLM-6-Turbo" + assert f"wandb/{model}" not in litellm.model_cost + + supported = litellm.get_supported_openai_params(model=f"wandb/{model}") + assert supported is not None + assert "reasoning_effort" in supported + + optional_params = litellm.utils.get_optional_params( + model=model, + custom_llm_provider="wandb", + reasoning_effort="medium", + drop_params=False, + ) + assert optional_params["reasoning_effort"] == "medium" diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index 481a52bebad..dd0d1bdbb9d 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -249,7 +249,6 @@ class TestWandbConfig: "model,explicit_false", [ ("meta-llama/Llama-3.1-8B-Instruct", False), - ("unknown-model", False), ("openai/gpt-oss-20b", True), ], ) @@ -290,3 +289,28 @@ class TestWandbConfig: supported_params = litellm.get_supported_openai_params(model=f"wandb/{model}") assert supported_params is not None assert "reasoning_effort" not in supported_params + + @pytest.mark.respx() + def test_wandb_completion_keeps_reasoning_effort_for_an_unregistered_model( + self, wandb_test_config, wandb_request_mock: respx.Route + ): + """A wandb id the registry has not named yet resolves through the + wandb-reasoning-baseline fallback generalization, so its reasoning_effort reaches + the provider instead of raising. W&B adds reasoning models faster than this + registry names them, and an exact entry still wins wherever one exists.""" + model: Final = "zai-org/GLM-6-Turbo" + assert f"wandb/{model}" not in litellm.model_cost + + completion( + model=f"wandb/{model}", + messages=[{"role": "user", "content": "Hello"}], + api_key="fake-wandb-key", + api_base="https://api.inference.wandb.ai/v1", + reasoning_effort="medium", + drop_params=False, + ) + + assert wandb_request_mock.call_count == 1 + request_body = json.loads(wandb_request_mock.calls[0].request.content) + assert request_body["model"] == model + assert request_body["reasoning_effort"] == "medium" From 7a1178a8595617d29453b20bd2e76fc629476999 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 14:36:22 -0700 Subject: [PATCH 383/411] fix(utils): stop model registration from shadowing capability rules Router writes every configured deployment into litellm.model_cost, and a deployment that declares no model_info lands there as an empty entry. An exact entry ends the model-info lookup ladder before the fallback generalizations are consulted, so that empty entry made the rules inert for the model: configuring one on a proxy stripped the capabilities the same model resolves to off-proxy. Seed a new registration from the capability rules its key matches. The caller's own model_info still wins field by field, so an explicit supports_reasoning: false on the deployment keeps overriding the rule. Claude-Session: https://claude.ai/code/session_01A6SkwJdfZUmkzfUkrEkqX8 --- litellm/utils.py | 5 +- .../test_fallback_generalizations.py | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index b4a2a29239b..69ee730ff30 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3108,7 +3108,10 @@ def register_model( existing_model = cast(dict, builtin_model_info) model_cost_key = existing_model["key"] else: - existing_model = {} + # An exact entry ends the lookup ladder before the capability rules are + # consulted, so seed from them: otherwise registering an unmapped model + # shadows the very defaults it would have resolved to unregistered. + existing_model = dict(match_capability_generalizations(_key_str) or {}) # mutable-ok: merge target model_cost_key = key builtin_entry = _resolve_builtin_model_cost_entry(key=_key_str, provider=provider) if builtin_entry is not None: diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 56552ec20ab..b0220a36054 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -631,3 +631,49 @@ def test_shipped_wandb_rule_keeps_reasoning_effort_on_an_unmapped_model(shipped_ drop_params=False, ) assert optional_params["reasoning_effort"] == "medium" + + +def test_router_registration_does_not_shadow_shipped_rules(shipped_cost_map): + """Regression: Router writes every configured deployment into ``litellm.model_cost``, + and an exact entry ends the lookup ladder before the rules are consulted. Registering + an unmapped model has to carry the rule defaults forward, or configuring a model on a + proxy silently strips the capabilities the same model resolves to off-proxy.""" + from litellm import Router + + unmapped_wandb = "wandb/zai-org/GLM-6-Turbo" + unmapped_claude = "anthropic/claude-opus-9" + assert unmapped_wandb not in litellm.model_cost + assert unmapped_claude not in litellm.model_cost + + Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": name, "api_key": "fake"}} + for name in (unmapped_wandb, unmapped_claude) + ] + ) + + assert unmapped_wandb in litellm.model_cost + assert unmapped_claude in litellm.model_cost + assert litellm.supports_reasoning(model="zai-org/GLM-6-Turbo", custom_llm_provider="wandb") is True + assert litellm.supports_reasoning(model="claude-opus-9", custom_llm_provider="anthropic") is True + + +def test_deployment_model_info_beats_the_seeded_rule_defaults(shipped_cost_map): + """Seeding a registration from the rules is a floor, not an override: an explicit + model_info on the deployment still wins, so a non-reasoning model can be configured + under a reasoning-first namespace.""" + from litellm import Router + + model = "wandb/some-org/NoThink-1" + Router( + model_list=[ + { + "model_name": model, + "litellm_params": {"model": model, "api_key": "fake"}, + "model_info": {"supports_reasoning": False}, + } + ] + ) + + assert litellm.model_cost[model]["supports_reasoning"] is False + assert litellm.supports_reasoning(model="some-org/NoThink-1", custom_llm_provider="wandb") is False From 61b0def8670c7edce4c4c9bd6a324b3827deb565 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 10 Sep 2026 16:51:37 -0700 Subject: [PATCH 384/411] test(ocr): isolate native Rust bridge contracts (#40410) * test(rust): add retained callback suite as expected failures * fix(tests): narrow retained callback xfails * test(rust): clarify retained callback contracts * test(ocr): clarify retained Rust contracts * test(ocr): restore guardrail contracts * test(ocr): require Rust file input parity * test(ocr): isolate native bridge contracts * fix(ci): repair Rust dispatch and OSV checks * test(ocr): assert explicit backend dispatch --- .github/workflows/test-rust.yml | 28 + tests/test_litellm_rust/README.md | 13 + tests/test_litellm_rust/conftest.py | 134 ++++- tests/test_litellm_rust/ocr/__init__.py | 1 + tests/test_litellm_rust/ocr/test_callbacks.py | 513 ++++++++++++++++++ tests/test_litellm_rust/ocr/test_dispatch.py | 44 ++ .../test_litellm_rust/ocr/test_guardrails.py | 74 +++ tests/test_litellm_rust/ocr/test_requests.py | 434 +++++++++++++++ tests/test_litellm_rust/support/__init__.py | 1 + .../support/callback_recorder.py | 101 ++++ .../support/recording_server.py | 108 ++++ tests/test_litellm_rust/support/requests.py | 59 ++ tests/test_litellm_rust/test_ocr.py | 61 ++- 13 files changed, 1533 insertions(+), 38 deletions(-) create mode 100644 tests/test_litellm_rust/README.md create mode 100644 tests/test_litellm_rust/ocr/__init__.py create mode 100644 tests/test_litellm_rust/ocr/test_callbacks.py create mode 100644 tests/test_litellm_rust/ocr/test_dispatch.py create mode 100644 tests/test_litellm_rust/ocr/test_guardrails.py create mode 100644 tests/test_litellm_rust/ocr/test_requests.py create mode 100644 tests/test_litellm_rust/support/__init__.py create mode 100644 tests/test_litellm_rust/support/callback_recorder.py create mode 100644 tests/test_litellm_rust/support/recording_server.py create mode 100644 tests/test_litellm_rust/support/requests.py diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index c6901411167..17b6481a2bf 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -4,8 +4,22 @@ on: push: paths: - "litellm-rust/**" + - "litellm/rust_bridge/**" + - "tests/test_litellm_rust/**" + - "litellm/integrations/custom_logger.py" + - "litellm/litellm_core_utils/litellm_logging.py" + - "litellm/litellm_core_utils/logging_worker.py" + - "litellm/proxy/guardrails/**" + - "litellm/utils.py" + - "litellm/ocr/**" + - "litellm/llms/base_llm/ocr/**" + - "litellm/llms/custom_httpx/llm_http_handler.py" + - "tests/test_litellm/ocr/**" + - "tests/test_litellm/conftest.py" + - "Makefile" - ".cargo/**" - "pyproject.toml" + - "uv.lock" - "rust-toolchain.toml" - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" @@ -20,8 +34,22 @@ on: - "litellm_**" paths: - "litellm-rust/**" + - "litellm/rust_bridge/**" + - "tests/test_litellm_rust/**" + - "litellm/integrations/custom_logger.py" + - "litellm/litellm_core_utils/litellm_logging.py" + - "litellm/litellm_core_utils/logging_worker.py" + - "litellm/proxy/guardrails/**" + - "litellm/utils.py" + - "litellm/ocr/**" + - "litellm/llms/base_llm/ocr/**" + - "litellm/llms/custom_httpx/llm_http_handler.py" + - "tests/test_litellm/ocr/**" + - "tests/test_litellm/conftest.py" + - "Makefile" - ".cargo/**" - "pyproject.toml" + - "uv.lock" - "rust-toolchain.toml" - ".github/actions/setup-uv-with-retries/**" - ".github/scripts/smoke_test_native_wheel.py" diff --git a/tests/test_litellm_rust/README.md b/tests/test_litellm_rust/README.md new file mode 100644 index 00000000000..4c117fb846b --- /dev/null +++ b/tests/test_litellm_rust/README.md @@ -0,0 +1,13 @@ +# Rust OCR bridge tests + +This suite covers OCR requests through LiteLLM's compiled Rust extension. OCR behavior tests live under `ocr/`; reusable OCR request, callback, and recording-server fixtures live under `support/` + +A test name identifies the OCR entrypoint or callback under test and its expected observable result. Parameter IDs state the execution mode or credential case. Keep multiple assertions together only when they prove one request, mutation, failure, or callback lifecycle behavior. Record callback observations and assert them after the callback returns because production logging can swallow callback exceptions + +`ocr/test_requests.py` covers provider payloads, file preparation, endpoint and credential resolution, normalized responses, errors, timeouts, and Azure token-provider behavior. `ocr/test_callbacks.py` covers OCR callback inputs, mutations, ordering, context, failure handling, concurrency, and cleanup. `ocr/test_guardrails.py` covers OCR post-call blocking and response replacement. These contract modules call the Rust bridge directly. `ocr/test_dispatch.py` has the single public API dispatch test, covering enabled native dispatch and disabled Python dispatch. `test_ocr.py` is a strict smoke test of the compiled Rust OCR transport + +Run `make test-rust-extension` as the acceptance command. It builds a fresh wheel, installs that wheel into a temporary environment, requires `LITELLM_RUST=1`, and runs this suite with isolated Python imports + +Collection fails when `LITELLM_RUST=1` is set but the compiled `_native` module cannot be imported. The autouse fixture isolates callback and configuration state but does not select a backend. Native contract tests call `litellm.rust_bridge.ocr` directly, while the strict dispatch test explicitly enables and disables Rust and records which OCR entrypoint runs + +The OCR contract modules are non-strict expected failures until the retained callback implementation from #40070 lands. The public dispatch test remains strict. Passing contract cases appear as XPASS so staging coverage stays visible diff --git a/tests/test_litellm_rust/conftest.py b/tests/test_litellm_rust/conftest.py index 02d274bb405..b0c75d9d2f5 100644 --- a/tests/test_litellm_rust/conftest.py +++ b/tests/test_litellm_rust/conftest.py @@ -1,24 +1,132 @@ +import asyncio import os +from collections.abc import AsyncIterator, Generator, Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack, contextmanager +from types import ModuleType +from typing import Final, cast import pytest +import pytest_asyncio + +import litellm +from litellm import utils +from litellm.litellm_core_utils import litellm_logging +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.rust_bridge.configuration import ( # pyright: ignore[reportPrivateUsage] # preserve raw configuration state in test isolation + _CONFIGURATION, + _parse_env_bool, +) +from tests.test_litellm_rust.support.callback_recorder import drain_logging +from tests.test_litellm_rust.support.recording_server import RecordingServer, recording_service + +CALLBACK_ATTRIBUTES: Final = ( + "callbacks", + "input_callback", + "success_callback", + "failure_callback", + "_async_input_callback", + "_async_success_callback", + "_async_failure_callback", +) +EXPECTED_FAILURE_REASONS: Final = { + "ocr/test_callbacks.py": "requires the OCR callback lifecycle implementation from #40070", + "ocr/test_guardrails.py": "requires the OCR guardrail lifecycle implementation from #40070", + "ocr/test_requests.py": "requires the OCR request and Azure authentication implementation from #40070", +} -def pytest_collection_modifyitems(items): - rust_enabled = os.environ.get("LITELLM_RUST", "").strip().lower() in { - "1", - "true", - "yes", - "on", - } - if not rust_enabled: - skip = pytest.mark.skip(reason="requires LITELLM_RUST=1 and a compiled Rust extension") +def _list_attribute(container: ModuleType, attribute: str) -> list[object]: + value: Final = getattr(container, attribute) + if not isinstance(value, list): + raise AssertionError(f"{container.__name__}.{attribute} is not a list") + return cast(list[object], value) + + +@contextmanager +def _isolated_list(container: ModuleType, attribute: str) -> Iterator[None]: + source: Final = _list_attribute(container, attribute) + original: Final = list(source) + source.clear() # mutable-ok: test isolation mutates global registries by design + try: + yield + finally: + source.clear() + source.extend(original) + setattr(container, attribute, source) + + +@contextmanager +def _rebound(container: object, attribute: str, value: object) -> Iterator[None]: + original: Final[object] = getattr(container, attribute) + setattr(container, attribute, value) + try: + yield + finally: + setattr(container, attribute, original) + + +@pytest_asyncio.fixture(autouse=True, loop_scope="function") +async def isolate_ocr_test_state() -> AsyncIterator[None]: + with ExitStack() as stack: + for attribute in CALLBACK_ATTRIBUTES: + stack.enter_context(_isolated_list(litellm, attribute)) + stack.enter_context(_isolated_list(litellm_logging, "_in_memory_loggers")) # pyright: ignore[reportPrivateUsage] # no public callback-cache accessor + stack.enter_context(_rebound(utils, "callback_list", [])) # rebind-ok: isolate legacy callback registry + stack.enter_context(_rebound(litellm, "cache", None)) # test-quality-ok: isolate process-global cache + stack.enter_context(_rebound(_CONFIGURATION, "override", None)) + executor: Final = ThreadPoolExecutor(thread_name_prefix="rust-ocr-test-logging") + stack.enter_context(_rebound(utils, "executor", executor)) + try: + yield + finally: + try: + await drain_logging() + finally: + await asyncio.to_thread(executor.shutdown, wait=True) + await GLOBAL_LOGGING_WORKER.stop() + + +@pytest.fixture +def recording_server() -> Generator[RecordingServer]: + with recording_service() as server: + yield server + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + for item in items: + if "test_litellm_rust" not in item.path.parts: + continue + relative_path: Final = "/".join(item.path.parts[item.path.parts.index("test_litellm_rust") + 1 :]) + reason: Final = EXPECTED_FAILURE_REASONS.get(relative_path) + if reason is not None: + item.add_marker(pytest.mark.xfail(reason=reason, strict=False)) + + if not _parse_env_bool(os.environ.get("LITELLM_RUST")): + skip: Final = pytest.mark.skip(reason="requires LITELLM_RUST=1 and a compiled Rust extension") for item in items: - item.add_marker(skip) + if "test_litellm_rust" in item.path.parts: + item.add_marker(skip) return try: from litellm.rust_bridge import _native # noqa: F401 # validates the installed extension except ImportError as error: - raise pytest.UsageError( - "LITELLM_RUST=1 requires a compiled litellm.rust_bridge._native extension" - ) from error + raise pytest.UsageError("LITELLM_RUST=1 requires a compiled litellm.rust_bridge._native extension") from error + + +@pytest.fixture +def isolated_azure_auth(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "AZURE_AI_API_KEY", + "AZURE_AI_API_BASE", + "AZURE_AD_TOKEN", + "AZURE_TENANT_ID", + "AZURE_CLIENT_ID", + "AZURE_CLIENT_SECRET", + "AZURE_USERNAME", + "AZURE_PASSWORD", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "enable_azure_ad_token_refresh", False) diff --git a/tests/test_litellm_rust/ocr/__init__.py b/tests/test_litellm_rust/ocr/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/test_litellm_rust/ocr/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_litellm_rust/ocr/test_callbacks.py b/tests/test_litellm_rust/ocr/test_callbacks.py new file mode 100644 index 00000000000..b08446412c0 --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_callbacks.py @@ -0,0 +1,513 @@ +import asyncio +import copy +import queue +import threading +from typing import Final + +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger +from tests.test_litellm_rust.support.requests import ( + OCR_DOCUMENT, + OCR_RESPONSE, + call_native_aocr, + call_native_ocr, + request_body, + request_headers, +) +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +def call_native_ocr_with_callbacks(server: RecordingServer, callbacks: list[CustomLogger], **kwargs: object): + return call_native_ocr(server, callbacks=callbacks, **kwargs) + + +async def call_native_aocr_with_callbacks(server: RecordingServer, callbacks: list[CustomLogger], **kwargs: object): + return await call_native_aocr(server, callbacks=callbacks, **kwargs) + + +def test_native_ocr_pre_call_callback_receives_transformed_provider_request(ocr_server: RecordingServer) -> None: + observations: Final = [] + + class Observe(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + observations.append((model, copy.deepcopy(kwargs["additional_args"]))) + + call_native_ocr_with_callbacks(ocr_server, [Observe()], pages=[0]) + + assert len(observations) == 1 + model, additional_args = observations[0] + assert model == "mistral-ocr-latest" + assert additional_args["api_base"] == f"{ocr_server.base_url}/v1/ocr" + assert additional_args["complete_input_dict"] == { + "model": "mistral-ocr-latest", + "document": OCR_DOCUMENT, + "pages": [0], + } + + +@pytest.mark.parametrize("raise_after_edit", [False, True], ids=["callback-returns", "callback-raises"]) +def test_native_ocr_pre_call_body_edit_reaches_next_callback_and_provider( + ocr_server: RecordingServer, raise_after_edit: bool +) -> None: + observed: Final = [] + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + request_body(kwargs)["include_image_base64"] = True + if raise_after_edit: + raise RuntimeError("pre-call callback failed") + + class Observe(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + observed.append(copy.deepcopy(request_body(kwargs))) + + call_native_ocr_with_callbacks(ocr_server, [Edit(), Observe()], include_image_base64=False) + + assert observed[0]["include_image_base64"] is True + assert ocr_server.requests[0].body["include_image_base64"] is True + + +def test_native_ocr_pre_call_header_edit_reaches_next_callback_and_provider(ocr_server: RecordingServer) -> None: + observed: Final = [] + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + request_headers(kwargs)["x-audit-tag"] = "reviewed" + + class Observe(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + observed.append(dict(request_headers(kwargs))) + + call_native_ocr_with_callbacks(ocr_server, [Edit(), Observe()]) + + assert observed[0]["x-audit-tag"] == "reviewed" + assert ocr_server.requests[0].headers["x-audit-tag"] == "reviewed" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_ocr_pre_call_nested_document_edit_updates_caller_callback_and_provider_references( + ocr_server: RecordingServer, asynchronous: bool +) -> None: + original: Final = dict(OCR_DOCUMENT) + replacement_url: Final = "data:application/pdf;base64,ZGVm" + retained: Final = [] + aliases: Final = [] + + class Retain(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + aliases.append(request_body(kwargs)["document"] is original) + retained.append(request_body(kwargs)["document"]) + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + original["document_url"] = replacement_url + + arguments: Final = { + "model": "mistral/mistral-ocr-latest", + "document": original, + "api_key": "test-key", + "api_base": ocr_server.base_url, + "callbacks": [Retain(), Edit()], + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) + if asynchronous + else call_native_ocr(ocr_server, **arguments) + ) + + assert aliases == [True] + assert retained[0]["document_url"] == replacement_url + assert original["document_url"] == replacement_url + assert ocr_server.requests[0].body["document"]["document_url"] == replacement_url + assert response.pages[0].markdown == "native OCR response" + + +def test_native_ocr_pre_call_document_replacement_does_not_mutate_original_document( + ocr_server: RecordingServer, +) -> None: + original: Final = dict(OCR_DOCUMENT) + replacement: Final = {"type": "document_url", "document_url": "data:application/pdf;base64,ZGVm"} + retained: Final = [] + + class RetainAndReplace(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + body = request_body(kwargs) + retained.append(body["document"]) + body["document"] = replacement + + call_native_ocr( + ocr_server, + document=original, + callbacks=[RetainAndReplace()], + ) + + assert retained[0] is original + assert original["document_url"] == OCR_DOCUMENT["document_url"] + assert ocr_server.requests[0].body["document"] == replacement + + +def test_native_ocr_pre_call_body_rebinding_is_visible_to_callbacks_but_not_provider( + ocr_server: RecordingServer, +) -> None: + observed: Final = [] + + class Rebind(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + kwargs["additional_args"]["complete_input_dict"] = {"replacement": True} + + class Observe(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + observed.append(request_body(kwargs)) + + call_native_ocr_with_callbacks(ocr_server, [Rebind(), Observe()]) + + assert observed == [{"replacement": True}] + assert ocr_server.requests[0].body == {"model": "mistral-ocr-latest", "document": OCR_DOCUMENT} + + +def test_native_ocr_callback_retained_body_observes_later_callback_mutation(ocr_server: RecordingServer) -> None: + queued: Final = [] + + class QueuePayload(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + queued.append(request_body(kwargs)) + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + request_body(kwargs)["queued-edit"] = True + + call_native_ocr_with_callbacks(ocr_server, [QueuePayload(), Edit()]) + + assert queued[0]["queued-edit"] is True + + +def test_native_ocr_success_callback_receives_state_added_by_pre_call_callback(ocr_server: RecordingServer) -> None: + token: Final = object() + terminal_tokens: queue.SimpleQueue[object] = queue.SimpleQueue() + finished: Final = threading.Event() + + class Stash(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + kwargs["test-token"] = token + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + terminal_tokens.put(kwargs["test-token"]) + finished.set() + + call_native_ocr_with_callbacks(ocr_server, [Stash()]) + + assert finished.wait(10) + assert terminal_tokens.get_nowait() is token + + +@pytest.mark.asyncio +async def test_native_aocr_success_callback_receives_call_id_metadata_and_response( + ocr_server: RecordingServer, +) -> None: + recorder: Final = RecordingLogger() + + await call_native_aocr_with_callbacks( + ocr_server, + [recorder], + litellm_call_id="ocr-success", + metadata={"source": "callback-test"}, + ) + events: Final = await recorder.wait_for_async("async_log_success_event") + + assert len(events) == 1 + assert events[0].call_type == "aocr" + assert events[0].kwargs["litellm_call_id"] == "ocr-success" + assert events[0].kwargs["litellm_params"]["metadata"]["source"] == "callback-test" + assert events[0].response.pages[0].markdown == "native OCR response" + + +@pytest.mark.asyncio +async def test_native_aocr_failure_callbacks_receive_call_type_error_and_no_response( + ocr_server: RecordingServer, +) -> None: + ocr_server.enqueue(ResponseSpec(body={"message": "provider unavailable"}, status=500)) + observations: Final = [] + + class Observe(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + observations.append(("sync", kwargs["call_type"], kwargs["exception"], response_obj)) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + observations.append(("async", kwargs["call_type"], kwargs["exception"], response_obj)) + + with pytest.raises(litellm.InternalServerError): + await call_native_aocr_with_callbacks(ocr_server, [Observe()]) + + assert [observation[0] for observation in observations] == ["sync", "async"] + assert all(observation[1] == "aocr" for observation in observations) + assert all(isinstance(observation[2], litellm.InternalServerError) for observation in observations) + assert all(observation[3] is None for observation in observations) + + +@pytest.mark.asyncio +async def test_native_aocr_pre_call_callback_runs_on_caller_loop_and_thread(ocr_server: RecordingServer) -> None: + caller_loop: Final = asyncio.get_running_loop() + caller_thread: Final = threading.current_thread() + recorder: Final = RecordingLogger() + + await call_native_aocr_with_callbacks(ocr_server, [recorder]) + + events: Final = await recorder.wait_for_async("log_pre_api_call") + assert len(events) == 1 + assert events[0].loop is caller_loop + assert events[0].thread is caller_thread + + +@pytest.mark.asyncio +async def test_native_aocr_failure_callbacks_receive_state_added_by_pre_call_callback( + ocr_server: RecordingServer, +) -> None: + ocr_server.enqueue(ResponseSpec(body={"message": "provider unavailable"}, status=500)) + token: Final = object() + observed: Final = [] + + class TrackInFlightRequest(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + kwargs["request-token"] = token + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + observed.append(("sync", kwargs["request-token"])) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + observed.append(("async", kwargs["request-token"])) + + with pytest.raises(litellm.InternalServerError): + await call_native_aocr_with_callbacks(ocr_server, [TrackInFlightRequest()]) + + assert [event for event, _ in observed] == ["sync", "async"] + assert all(observed_token is token for _, observed_token in observed) + + +@pytest.mark.asyncio +async def test_native_aocr_callback_error_does_not_mask_provider_error_or_skip_later_failure_callbacks( + ocr_server: RecordingServer, +) -> None: + ocr_server.enqueue(ResponseSpec(body={"message": "provider unavailable"}, status=500)) + recorder: Final = RecordingLogger() + + class FailingCallback(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + raise RuntimeError("failure callback failed") + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + raise RuntimeError("failure callback failed") + + with pytest.raises(litellm.InternalServerError) as caught: + await call_native_aocr_with_callbacks(ocr_server, [FailingCallback(), recorder]) + + sync_events: Final = tuple(event for event in recorder.events if event.name == "log_failure_event") + async_events: Final = tuple(event for event in recorder.events if event.name == "async_log_failure_event") + assert len(sync_events) == 1 + assert len(async_events) == 1 + assert sync_events[0].kwargs["exception"] is caught.value + assert async_events[0].kwargs["exception"] is caught.value + assert "async_log_success_event" not in recorder.names + + +def test_native_ocr_dispatches_each_callback_phase_once_when_logger_is_registered_multiple_times( + ocr_server: RecordingServer, +) -> None: + recorder: Final = RecordingLogger() + + call_native_ocr_with_callbacks( + ocr_server, + [recorder, recorder], + success_callback=[recorder], + failure_callback=[recorder], + ) + recorder.wait_for("log_success_event") + + assert recorder.names.count("log_pre_api_call") == 1 + assert recorder.names.count("logging_hook") == 1 + assert recorder.names.count("log_success_event") == 1 + assert "log_failure_event" not in recorder.names + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_azure_ocr_resolves_token_before_pre_call_on_caller_context( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, +) -> None: + from contextvars import ContextVar + context: Final = ContextVar("azure-token-context", default="missing") + context.set("caller") + caller_thread: Final = threading.current_thread() + caller_loop: Final = asyncio.get_running_loop() + observations: Final = [] + + class Provider: + def __call__(self) -> str: + assert context.get() == "caller" + assert threading.current_thread() is caller_thread + assert asyncio.get_running_loop() is caller_loop + observations.append("token") + return "caller-token" + + class Edit(CustomLogger): + def log_pre_api_call(self, model, _messages, kwargs): + assert request_headers(kwargs)["Authorization"] == "Bearer caller-token" + observations.append("pre_call") + request_headers(kwargs)["Authorization"] = "Bearer edited" + + provider: Final = Provider() + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": provider, + "callbacks": [Edit()], + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) + if asynchronous + else call_native_ocr(ocr_server, **arguments) + ) + assert response.pages[0].markdown == "native OCR response" + assert observations == ["token", "pre_call"] + assert ocr_server.requests[0].headers["authorization"] == "Bearer edited" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_azure_ocr_token_provider_can_make_nested_native_ocr_call( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, +) -> None: + ocr_server.expected_requests = 2 + calls: Final = [] + + def provider() -> str: + calls.append("token") + nested: Final = call_native_ocr(ocr_server) + assert nested.pages[0].markdown == "native OCR response" + return "outer-token" + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": provider, + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) + if asynchronous + else call_native_ocr(ocr_server, **arguments) + ) + assert response.pages[0].markdown == "native OCR response" + assert calls == ["token"] + assert [request.headers["authorization"] for request in ocr_server.requests] == [ + "Bearer test-key", + "Bearer outer-token", + ] + + +@pytest.mark.asyncio +async def test_concurrent_native_azure_ocr_calls_isolate_token_results_and_error( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + ocr_server.expected_requests = 2 + + async def request(token: str, fail: bool) -> object: + def provider() -> str: + if fail: + raise ValueError(token) + return token + + return await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token_provider=provider, + ) + + responses: Final = await asyncio.gather( + request("first", False), + request("failed", True), + request("second", False), + return_exceptions=True, + ) + assert isinstance(responses[0], OCRResponse) + assert isinstance(responses[1], litellm.APIConnectionError) + assert "Failed to get Azure AD token: failed" in str(responses[1]) + assert isinstance(responses[2], OCRResponse) + assert sorted(request.headers["authorization"] for request in ocr_server.requests) == [ + "Bearer first", + "Bearer second", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["success", "failure", "cancellation"]) +async def test_native_azure_ocr_releases_token_provider_after_terminal_outcome( + ocr_server: RecordingServer, + isolated_azure_auth: None, + outcome: str, +) -> None: + import gc + import weakref + from tests.test_litellm_rust.support.callback_recorder import drain_logging + class Provider: + def __call__(self) -> str: + if outcome == "failure": + raise ValueError("unavailable") + return "caller-token" + + async def invoke() -> weakref.ReferenceType[Provider]: + provider: Final = Provider() + reference: Final = weakref.ref(provider) + if outcome == "failure": + ocr_server.expected_requests = 0 + with pytest.raises(litellm.APIConnectionError): + await call_native_aocr( + ocr_server, model="azure_ai/mistral-ocr-latest", api_key=None, azure_ad_token_provider=provider + ) + elif outcome == "cancellation": + ocr_server.enqueue(ResponseSpec(body=OCR_RESPONSE, delay=0.1)) + task: Final = asyncio.create_task( + call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token_provider=provider, + ) + ) + await ocr_server.wait_for_requests(1) + assert reference() is provider + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + response: Final = await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token_provider=provider, + ) + assert response.pages[0].markdown == "native OCR response" + return reference + + reference: Final = await invoke() + await drain_logging() + await asyncio.sleep(0) + gc.collect() + assert reference() is None diff --git a/tests/test_litellm_rust/ocr/test_dispatch.py b/tests/test_litellm_rust/ocr/test_dispatch.py new file mode 100644 index 00000000000..a6c76bc5d0e --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_dispatch.py @@ -0,0 +1,44 @@ +from typing import Final +from unittest.mock import Mock + +import pytest + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.ocr import main as ocr_main +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import OCR_DOCUMENT, OCR_MODEL, OCR_RESPONSE + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +@pytest.mark.parametrize("rust_enabled", [True, False], ids=["enabled", "disabled"]) +def test_public_ocr_dispatches_according_to_rust_setting( + ocr_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + rust_enabled: bool, +) -> None: + rust_call: Final = Mock(wraps=ocr_main.rust_ocr_bridge.ocr) + python_call: Final = Mock(wraps=ocr_main.base_llm_http_handler.ocr) + monkeypatch.setattr(ocr_main.rust_ocr_bridge, "ocr", rust_call) + monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", python_call) + litellm.rust(rust_enabled) + + response: Final = litellm.ocr( + model=OCR_MODEL, + document=OCR_DOCUMENT, + api_key="test-key", + api_base=ocr_server.base_url, + ) + + assert isinstance(response, OCRResponse) + assert response.pages[0].markdown == "native OCR response" + assert rust_call.call_count == int(rust_enabled) + assert python_call.call_count == int(not rust_enabled) + assert len(ocr_server.requests) == 1 diff --git a/tests/test_litellm_rust/ocr/test_guardrails.py b/tests/test_litellm_rust/ocr/test_guardrails.py new file mode 100644 index 00000000000..f6fc1c7cb8d --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_guardrails.py @@ -0,0 +1,74 @@ +from typing import Final + +import pytest +from fastapi import HTTPException + +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail +from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks +from litellm.types.utils import CallTypes +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import OCR_RESPONSE, call_native_aocr + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +class ReplaceOCRMarkdown(CustomGuardrail): + def __init__(self) -> None: + super().__init__( + guardrail_name="replace-ocr-markdown", event_hook=GuardrailEventHooks.post_call, default_on=True + ) + self.call_types: list[CallTypes] = [] + + async def async_post_call_success_deployment_hook(self, request_data, response, call_type): + self.call_types.append(call_type) + reviewed_page: Final = response.pages[0].model_copy(update={"markdown": "Reviewed OCR"}) + return response.model_copy(update={"pages": [reviewed_page]}) + + +@pytest.mark.asyncio +async def test_native_aocr_post_call_content_filter_blocks_matching_markdown( + ocr_server: RecordingServer, +) -> None: + guardrail: Final = ContentFilterGuardrail( + guardrail_name="block-native-ocr-markdown", + event_hook=GuardrailEventHooks.post_call, + blocked_words=[BlockedWord(keyword="native OCR response", action=ContentFilterAction.BLOCK)], + ) + litellm.callbacks.append(guardrail) + + with pytest.raises(HTTPException, match="Content blocked") as blocked: + await call_native_aocr(ocr_server, guardrails=[guardrail.guardrail_name]) + + assert blocked.value.status_code == 400 + assert len(ocr_server.requests) == 1 + + +@pytest.mark.asyncio +async def test_native_aocr_post_call_replacement_reaches_caller_and_success_callback( + ocr_server: RecordingServer, +) -> None: + guardrail: Final = ReplaceOCRMarkdown() + recorder: Final = RecordingLogger() + litellm.callbacks.append(guardrail) + + response: Final = await call_native_aocr( + ocr_server, + callbacks=[recorder], + guardrails=[guardrail.guardrail_name], + ) + success_events: Final = await recorder.wait_for_async("async_log_success_event") + + assert guardrail.call_types == [CallTypes.aocr] + assert response.pages[0].markdown == "Reviewed OCR" + assert len(success_events) == 1 + assert success_events[0].response.pages[0].markdown == "Reviewed OCR" + assert "guardrails" not in ocr_server.requests[0].body diff --git a/tests/test_litellm_rust/ocr/test_requests.py b/tests/test_litellm_rust/ocr/test_requests.py new file mode 100644 index 00000000000..d241fe08fc8 --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_requests.py @@ -0,0 +1,434 @@ +from typing import Final + +import pytest + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger +from tests.test_litellm_rust.support.requests import ( + OCR_DOCUMENT, + OCR_RESPONSE, + call_native_aocr, + call_native_ocr, +) +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_azure_ocr_uses_token_provider_result_as_bearer_token( + ocr_server: RecordingServer, isolated_azure_auth: None, asynchronous: bool +) -> None: + calls: Final = [] + + def token_provider() -> str: + calls.append("token") + return "callback-token" + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": token_provider, + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) + ) + + assert calls == ["token"] + assert response.pages[0].markdown == "native OCR response" + assert_native_request(ocr_server) + assert ocr_server.requests[0].headers["authorization"] == "Bearer callback-token" + + +@pytest.fixture +def ocr_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=OCR_RESPONSE) + return recording_server + + +def assert_native_request(server: RecordingServer) -> None: + assert len(server.requests) == 1 + assert not server.requests[0].headers.get("user-agent", "").startswith("python-httpx") + + +def test_native_ocr_sends_model_and_document_to_mistral_ocr_path(ocr_server: RecordingServer) -> None: + response: Final = call_native_ocr(ocr_server) + + assert response.pages[0].markdown == "native OCR response" + assert_native_request(ocr_server) + assert ocr_server.requests[0].path == "/v1/ocr" + assert ocr_server.requests[0].body == {"model": "mistral-ocr-latest", "document": OCR_DOCUMENT} + + +def test_native_ocr_prepares_file_document_like_python(ocr_server: RecordingServer) -> None: + response: Final = call_native_ocr( + ocr_server, + document={"type": "file", "file": b"%PDF-1.4", "mime_type": "application/pdf"}, + ) + + assert response.pages[0].markdown == "native OCR response" + assert_native_request(ocr_server) + assert ocr_server.requests[0].body == { + "model": "mistral-ocr-latest", + "document": { + "type": "document_url", + "document_url": "data:application/pdf;base64,JVBERi0xLjQ=", + }, + } + + +def test_native_ocr_sends_pages_and_image_options(ocr_server: RecordingServer) -> None: + call_native_ocr(ocr_server, pages=[0, 2], include_image_base64=True) + + assert ocr_server.requests[0].body["pages"] == [0, 2] + assert ocr_server.requests[0].body["include_image_base64"] is True + + +def test_native_ocr_merges_custom_headers_with_authorization(ocr_server: RecordingServer) -> None: + call_native_ocr(ocr_server, extra_headers={"x-trace-id": "trace-1"}) + + assert ocr_server.requests[0].headers["authorization"] == "Bearer test-key" + assert ocr_server.requests[0].headers["x-trace-id"] == "trace-1" + + +def test_native_mistral_ocr_uses_environment_api_key_when_argument_is_missing( + ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("MISTRAL_API_KEY", "environment-key") + + call_native_ocr(ocr_server, api_key=None) + + assert ocr_server.requests[0].headers["authorization"] == "Bearer environment-key" + + +def test_native_mistral_ocr_prefers_explicit_api_key_over_environment( + ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("MISTRAL_API_KEY", "environment-key") + + call_native_ocr(ocr_server) + + assert ocr_server.requests[0].headers["authorization"] == "Bearer test-key" + + +def test_native_azure_ocr_uses_environment_endpoint_and_api_key( + ocr_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("AZURE_AI_API_KEY", "azure-key") + monkeypatch.setenv("AZURE_AI_API_BASE", ocr_server.base_url) + + call_native_ocr(ocr_server, model="azure_ai/pixtral-12b-2409", api_key=None, api_base=None) + + assert_native_request(ocr_server) + assert ocr_server.requests[0].path == "/providers/mistral/azure/ocr" + assert ocr_server.requests[0].headers["authorization"] == "Bearer azure-key" + + +def test_native_vertex_ocr_builds_path_from_project_and_location(ocr_server: RecordingServer) -> None: + call_native_ocr( + ocr_server, + model="vertex_ai/mistral-ocr-2505", + api_key="vertex-token", + vertex_project="project-1", + vertex_location="us-central1", + ) + + assert_native_request(ocr_server) + assert ocr_server.requests[0].path == ( + "/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-2505:rawPredict" + ) + + +def test_native_ocr_normalizes_provider_response_model_and_usage(ocr_server: RecordingServer) -> None: + response: Final = call_native_ocr(ocr_server) + + assert isinstance(response, OCRResponse) + assert response.model == "mistral-ocr-latest" + assert response.usage_info.pages_processed == 1 + + +def test_native_ocr_maps_provider_400_without_exposing_response_body(ocr_server: RecordingServer) -> None: + ocr_server.enqueue(ResponseSpec(body={"message": "invalid OCR request"}, status=400)) + + with pytest.raises(litellm.BadRequestError) as caught: + call_native_ocr(ocr_server) + + assert caught.value.status_code == 400 + assert caught.value.model == "mistral-ocr-latest" + assert caught.value.llm_provider == "mistral" + assert "invalid OCR request" not in str(caught.value) + + +def test_native_ocr_raises_transport_error_when_request_exceeds_timeout(ocr_server: RecordingServer) -> None: + ocr_server.enqueue(ResponseSpec(body=OCR_RESPONSE, delay=0.2)) + + with pytest.raises(RuntimeError, match="OCR transport failed"): + call_native_ocr(ocr_server, timeout=0.01) + + assert len(ocr_server.requests) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "credentials, expected_token, expected_calls", + [ + ({"api_key": "resource-key"}, "resource-key", 0), + ({"azure_ad_token": "static-token"}, "callback-1", 1), + ({"extra_headers": {"Authorization": "Bearer override"}}, "override", 1), + ], + ids=["api-key-skips-provider", "provider-overrides-static-token", "header-overrides-provider"], +) +async def test_native_azure_ocr_applies_python_credential_precedence( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, + credentials: dict[str, object], + expected_token: str, + expected_calls: int, +) -> None: + calls: Final = [] + + def token_provider() -> str: + calls.append("token") + return f"callback-{len(calls)}" + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": token_provider, + **credentials, + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) + ) + assert response.pages[0].markdown == "native OCR response" + assert len(calls) == expected_calls + assert len(ocr_server.requests) == 1 + assert ocr_server.requests[0].headers["authorization"] == f"Bearer {expected_token}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_azure_ocr_calls_token_provider_for_each_request( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, +) -> None: + calls: Final = [] + ocr_server.expected_requests = 2 + + def token_provider() -> str: + calls.append("token") + return f"callback-{len(calls)}" + + for _ in range(2): + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": token_provider, + } + response: Final = ( + await call_native_aocr(ocr_server, **arguments) + if asynchronous + else call_native_ocr(ocr_server, **arguments) + ) + assert response.pages[0].markdown == "native OCR response" + assert len(calls) == 2 + assert [request.headers["authorization"] for request in ocr_server.requests] == [ + "Bearer callback-1", + "Bearer callback-2", + ] + + +class TokenAbort(BaseException): + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "failure", + ["non_string", "type_error", "ordinary", "abort"], + ids=["non-string-result", "type-error", "value-error", "base-exception"], +) +async def test_native_azure_ocr_token_provider_failure_prevents_pre_call_callback_and_request( + ocr_server: RecordingServer, + isolated_azure_auth: None, + asynchronous: bool, + failure: str, +) -> None: + ocr_server.expected_requests = 0 + calls: Final = [] + recorder: Final = RecordingLogger() + original: Final = { + "type_error": TypeError("token type"), + "ordinary": ValueError("token unavailable"), + "abort": TokenAbort("abort"), + } + + def token_provider() -> object: + calls.append("token") + if failure == "non_string": + return 123 + raise original[failure] + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": token_provider, + "callbacks": [recorder], + } + expected: Final = TokenAbort if failure == "abort" else litellm.APIConnectionError + with pytest.raises(expected) as caught: + await call_native_aocr(ocr_server, **arguments) if asynchronous else call_native_ocr(ocr_server, **arguments) + assert calls == ["token"] + assert ocr_server.requests == [] + assert "log_pre_api_call" not in recorder.names + if failure == "ordinary": + assert "Failed to get Azure AD token: token unavailable" in str(caught.value) + assert isinstance(caught.value.__context__, RuntimeError) + assert caught.value.__context__.__cause__ is original[failure] + elif failure == "abort": + assert caught.value is original[failure] + elif failure == "type_error": + assert caught.value.__context__ is original[failure] + else: + assert isinstance(caught.value.__context__, TypeError) + + +@pytest.mark.parametrize( + "configuration", + [ + {"azure_ad_token": "oidc/assertion", "client_id": "client", "tenant_id": "tenant"}, + {"model": "azure_ai/doc-intelligence/prebuilt-read"}, + ], + ids=["oidc-assertion", "document-intelligence-model"], +) +def test_native_azure_ocr_rejects_unsupported_configuration_before_token_or_callbacks( + ocr_server: RecordingServer, + isolated_azure_auth: None, + configuration: dict[str, object], +) -> None: + ocr_server.expected_requests = 0 + calls: Final = [] + recorder: Final = RecordingLogger() + + def provider() -> str: + calls.append("token") + return "unused" + + arguments: Final = { + "model": "azure_ai/mistral-ocr-latest", + "api_key": None, + "azure_ad_token_provider": provider, + "callbacks": [recorder], + **configuration, + } + with pytest.raises(NotImplementedError): + call_native_ocr(ocr_server, **arguments) + assert calls == [] + assert recorder.events == () + assert ocr_server.requests == [] + + +@pytest.mark.asyncio +async def test_native_azure_ocr_validates_endpoint_before_calling_token_provider( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + ocr_server.expected_requests = 0 + calls: Final = [] + + def provider() -> str: + calls.append("token") + return "unused" + + with pytest.raises(litellm.APIConnectionError, match="Missing Azure AI API Base"): + await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + api_base=None, + azure_ad_token_provider=provider, + ) + assert calls == [] + assert ocr_server.requests == [] + + +@pytest.mark.asyncio +async def test_native_azure_ocr_does_not_fall_back_to_static_token_after_empty_provider_result( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + ocr_server.expected_requests = 0 + + def provider() -> str: + return "" + + with pytest.raises(litellm.APIConnectionError, match="Missing Azure AI credentials"): + await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token="static-token", + azure_ad_token_provider=provider, + ) + assert ocr_server.requests == [] + + +@pytest.mark.asyncio +async def test_native_azure_ocr_ignores_falsey_token_provider_and_uses_static_token( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + calls: Final = [] + + class Provider: + def __bool__(self) -> bool: + return False + + def __call__(self) -> str: + calls.append("token") + return "unused" + + response: Final = await call_native_aocr( + ocr_server, + model="azure_ai/mistral-ocr-latest", + api_key=None, + azure_ad_token="static-token", + azure_ad_token_provider=Provider(), + ) + assert response.pages[0].markdown == "native OCR response" + assert calls == [] + assert ocr_server.requests[0].headers["authorization"] == "Bearer static-token" + + +@pytest.mark.asyncio +async def test_native_azure_ocr_rejects_coroutine_returned_by_sync_token_provider( + ocr_server: RecordingServer, + isolated_azure_auth: None, +) -> None: + ocr_server.expected_requests = 0 + calls: Final = [] + + async def acquire() -> str: + calls.append("awaited") + return "unused" + + coroutine: Final = acquire() + + def provider() -> object: + return coroutine + + try: + with pytest.raises(litellm.APIConnectionError, match="Azure AD token must be a string"): + await call_native_aocr( + ocr_server, model="azure_ai/mistral-ocr-latest", api_key=None, azure_ad_token_provider=provider + ) + finally: + coroutine.close() + assert calls == [] + assert ocr_server.requests == [] diff --git a/tests/test_litellm_rust/support/__init__.py b/tests/test_litellm_rust/support/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/test_litellm_rust/support/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_litellm_rust/support/callback_recorder.py b/tests/test_litellm_rust/support/callback_recorder.py new file mode 100644 index 00000000000..6de011b1414 --- /dev/null +++ b/tests/test_litellm_rust/support/callback_recorder.py @@ -0,0 +1,101 @@ +import asyncio +import copy +import threading +import time +from dataclasses import dataclass +from typing import Final + +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker + + +async def drain_logging(worker: LoggingWorker = GLOBAL_LOGGING_WORKER) -> None: + await asyncio.sleep(0) + worker.start() + await asyncio.wait_for(worker.flush(), timeout=10) + + +@dataclass(frozen=True, slots=True) +class HookEvent: + name: str + call_type: str | None + thread: threading.Thread + loop: asyncio.AbstractEventLoop | None + kwargs: object + response: object + + +class RecordingLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self._events: list[HookEvent] = [] + self._condition = threading.Condition() + + @property + def events(self) -> tuple[HookEvent, ...]: + with self._condition: + return tuple(self._events) + + @property + def names(self) -> tuple[str, ...]: + return tuple(event.name for event in self.events) + + def _record(self, name: str, kwargs: object = None, response: object = None) -> None: + details: Final = kwargs if isinstance(kwargs, dict) else {} + try: + snapshot: Final = copy.deepcopy(details) + except Exception: + snapshot = dict(details) + if "exception" in details: + snapshot["exception"] = details["exception"] + try: + loop: Final = asyncio.get_running_loop() + except RuntimeError: + loop = None + event: Final = HookEvent( + name=name, + call_type=details.get("call_type"), + thread=threading.current_thread(), + loop=loop, + kwargs=snapshot, + response=response, + ) + with self._condition: + self._events.append(event) + self._condition.notify_all() + + def wait_for(self, name: str, count: int = 1, timeout: float = 10) -> tuple[HookEvent, ...]: + deadline: Final = time.monotonic() + timeout + with self._condition: + while sum(event.name == name for event in self._events) < count: + remaining: Final = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"Timed out waiting for {count} {name} events; saw {self.names}") + self._condition.wait(remaining) + return tuple(event for event in self._events if event.name == name) + + async def wait_for_async(self, name: str, count: int = 1, timeout: float = 10) -> tuple[HookEvent, ...]: + await asyncio.wait_for(asyncio.to_thread(self.wait_for, name, count, timeout), timeout=timeout + 1) + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=timeout) + return tuple(event for event in self.events if event.name == name) + + def log_pre_api_call(self, model, _messages, kwargs): + self._record("log_pre_api_call", kwargs) + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self._record("log_success_event", kwargs, response_obj) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self._record("async_log_success_event", kwargs, response_obj) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self._record("log_failure_event", kwargs, response_obj) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self._record("async_log_failure_event", kwargs, response_obj) + + def logging_hook(self, kwargs, result, call_type): + self._record("logging_hook", kwargs, result) + return kwargs, result diff --git a/tests/test_litellm_rust/support/recording_server.py b/tests/test_litellm_rust/support/recording_server.py new file mode 100644 index 00000000000..5a9b9497c6e --- /dev/null +++ b/tests/test_litellm_rust/support/recording_server.py @@ -0,0 +1,108 @@ +import asyncio +import copy +import json +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final + + +@dataclass +class RecordedRequest: + method: str + path: str + headers: dict[str, str] + raw_body: bytes + body: object | None + + +@dataclass +class ResponseSpec: + body: object + status: int = 200 + headers: dict[str, str] = field(default_factory=dict) + delay: float = 0 + + +@dataclass +class RecordingServer: + server: ThreadingHTTPServer + requests: list[RecordedRequest] + responses: list[ResponseSpec] + default_response: ResponseSpec + expected_requests: int | None = 1 + + @property + def base_url(self) -> str: + host, port = self.server.server_address + return f"http://{host}:{port}" + + def enqueue(self, response: ResponseSpec) -> None: + self.responses.append(response) + + async def wait_for_requests(self, count: int) -> None: + async with asyncio.timeout(2): + while len(self.requests) < count: + await asyncio.sleep(0.01) + + +@contextmanager +def recording_service() -> Iterator[RecordingServer]: + requests: list[RecordedRequest] = [] + responses: list[ResponseSpec] = [] + + class Handler(BaseHTTPRequestHandler): + def _handle(self) -> None: + content_length: Final = int(self.headers.get("Content-Length", "0")) + raw_body: Final = self.rfile.read(content_length) if content_length else b"" + body: Final = json.loads(raw_body) if raw_body else None + requests.append( + RecordedRequest( + method=self.command, + path=self.path, + headers={name.lower(): value for name, value in self.headers.items()}, + raw_body=raw_body, + body=body, + ) + ) + response: Final = responses.pop(0) if responses else copy.deepcopy(recording_server.default_response) + if response.delay: + time.sleep(response.delay) + payload: Final = json.dumps(response.body).encode() + self.send_response(response.status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + for name, value in response.headers.items(): + self.send_header(name, value) + self.end_headers() + try: + self.wfile.write(payload) + except (BrokenPipeError, ConnectionResetError): + pass + + do_POST = _handle + + def log_message(self, format: str, *args: object) -> None: + pass + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + try: + recording_server = RecordingServer( + server=server, + requests=requests, + responses=responses, + default_response=ResponseSpec(body={}), + ) + yield recording_server + finally: + server.shutdown() + server.server_close() + thread.join() + if recording_server.expected_requests is not None: + assert len(recording_server.requests) == recording_server.expected_requests + assert recording_server.responses == [] diff --git a/tests/test_litellm_rust/support/requests.py b/tests/test_litellm_rust/support/requests.py new file mode 100644 index 00000000000..d681d752ffd --- /dev/null +++ b/tests/test_litellm_rust/support/requests.py @@ -0,0 +1,59 @@ +from typing import Final + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.rust_bridge import ocr as native_ocr +from tests.test_litellm_rust.support.recording_server import RecordingServer + +OCR_DOCUMENT: Final = {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"} +OCR_MODEL: Final = "mistral/mistral-ocr-latest" +OCR_RESPONSE: Final = { + "pages": [{"index": 0, "markdown": "native OCR response", "images": [], "dimensions": None}], + "model": "mistral-ocr-latest", + "usage_info": {"pages_processed": 1, "doc_size_bytes": 3}, +} + + +def ocr_arguments(server: RecordingServer, **kwargs: object) -> dict[str, object]: + return { + "model": OCR_MODEL, + "document": dict(OCR_DOCUMENT), + "api_key": "test-key", + "api_base": server.base_url, + **kwargs, + } + + +def call_ocr(server: RecordingServer, **kwargs: object) -> OCRResponse: + response: Final = litellm.ocr(**ocr_arguments(server, **kwargs)) + if not isinstance(response, OCRResponse): + raise TypeError(f"Expected OCRResponse, got {type(response).__name__}") + return response + + +async def call_aocr(server: RecordingServer, **kwargs: object) -> OCRResponse: + return await litellm.aocr(**ocr_arguments(server, **kwargs)) + + +def call_native_ocr(server: RecordingServer, **kwargs: object) -> OCRResponse: + return native_ocr.ocr(ocr_arguments(server, **kwargs)) + + +async def call_native_aocr(server: RecordingServer, **kwargs: object) -> OCRResponse: + return await native_ocr.aocr(ocr_arguments(server, **kwargs)) + + +def request_body(kwargs: dict[str, object]) -> dict[str, object]: + additional_args = kwargs["additional_args"] + assert isinstance(additional_args, dict) + body = additional_args["complete_input_dict"] + assert isinstance(body, dict) + return body + + +def request_headers(kwargs: dict[str, object]) -> dict[str, object]: + additional_args = kwargs["additional_args"] + assert isinstance(additional_args, dict) + headers = additional_args["headers"] + assert isinstance(headers, dict) + return headers diff --git a/tests/test_litellm_rust/test_ocr.py b/tests/test_litellm_rust/test_ocr.py index d5b1fce1139..1293de9ee0e 100644 --- a/tests/test_litellm_rust/test_ocr.py +++ b/tests/test_litellm_rust/test_ocr.py @@ -1,31 +1,34 @@ import json import threading +from collections.abc import Generator +from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final import pytest -import litellm +from litellm.rust_bridge import ocr as rust_ocr_bridge pytestmark = pytest.mark.requires_rust_extension +@dataclass(frozen=True, slots=True) +class RecordedOCRRequest: + body: object + + @pytest.fixture -def ocr_server(): - requests = [] +def ocr_server() -> Generator[tuple[ThreadingHTTPServer, list[RecordedOCRRequest]]]: + requests: Final[list[RecordedOCRRequest]] = [] class Handler(BaseHTTPRequestHandler): - def do_POST(self): + def do_POST(self) -> None: requests.append( - { - "headers": {name.lower(): value for name, value in self.headers.items()}, - "body": json.loads(self.rfile.read(int(self.headers["Content-Length"]))), - } + RecordedOCRRequest( + body=json.loads(self.rfile.read(int(self.headers["Content-Length"]))), + ) ) - if self.headers.get("User-Agent", "").startswith("python-httpx"): - self.send_response(418) - self.end_headers() - return - response = json.dumps( + response: Final = json.dumps( { "pages": [{"index": 0, "markdown": "native OCR response", "images": [], "dimensions": None}], "model": "mistral-ocr-latest", @@ -38,11 +41,11 @@ def ocr_server(): self.end_headers() self.wfile.write(response) - def log_message(self, format, *args): + def log_message(self, format: str, *args: object) -> None: pass - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - thread = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True) + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True) thread.start() try: yield server, requests @@ -52,21 +55,29 @@ def ocr_server(): thread.join() -def test_ocr_with_rust_extension(ocr_server): +def test_native_ocr_with_compiled_rust_extension( + ocr_server: tuple[ThreadingHTTPServer, list[RecordedOCRRequest]], +) -> None: server, requests = ocr_server - host, port = server.server_address + address: Final = server.server_address + host: Final = str(address[0]) + port: Final = int(address[1]) - response = litellm.ocr( - model="mistral/mistral-ocr-latest", + response: Final = rust_ocr_bridge.ocr( + model="mistral-ocr-latest", document={"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, api_key="test-key", api_base=f"http://{host}:{port}", + custom_llm_provider="mistral", + extra_headers=None, + optional_params={}, + timeout=None, ) - assert response.pages[0].markdown == "native OCR response" + assert response is not None + assert response["pages"][0]["markdown"] == "native OCR response" assert len(requests) == 1 - assert not requests[0]["headers"].get("user-agent", "").startswith("python-httpx") - assert requests[0]["body"] == { - "model": "mistral-ocr-latest", - "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, + assert requests[0].body == { + "model": "mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, } From 760043b5337a83f0356cd717e5b555229980deec Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 16:57:53 -0700 Subject: [PATCH 385/411] fix(reset_budget_job): reset end users by budget link, not by user id The cascade zeroed end-user spend with a single update_many whose where clause enumerated every dependent user id. Prisma compiles that IN-list into one prepared statement carrying one bind variable per customer, and PostgreSQL caps a statement at 32,767 of them. Once a shared budget had more dependents than that the statement could not be parsed at all, so the atomic cascade rolled back, budget_reset_at never advanced, and the tier stayed due on every later tick forever. Customers sitting at their cap were blocked indefinitely with only a recurring log line to show for it. End users now match on budget_id like every other gated table, plus a NULL-budget_id branch for the implicitly created rows that carry no link and ride the default tier. The statement's bind count now tracks the number of expiring tiers rather than the customer population, so a reset costs the same whether a budget has ten dependents or a million. Fixes #40564 Claude-Session: https://claude.ai/code/session_01Hn5E8Jz1LjGLFyiYxBRcBW --- .../proxy/common_utils/reset_budget_job.py | 46 ++++++------ .../test_proxy_budget_reset.py | 6 +- .../common_utils/test_reset_budget_job.py | 75 ++++++++++++++++--- 3 files changed, 88 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index f2648c8466e..35e74418628 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -224,33 +224,31 @@ def _queue_budget_linked_resets( def _queue_enduser_resets(writes: LinkedSpendResetWrites, cascade: "_BudgetCascade") -> None: - """End users are matched by id rather than budget link: rows with no - budget_id ride the default budget tier (litellm.max_end_user_budget_id). - Zero-before-decrement ordering matters here too (see - _queue_budget_linked_resets).""" - if not cascade.rollover_caps: - if cascade.endusers: - writes.queue_spend_zero( - where={"user_id": {"in": [row.user_id for row in cascade.endusers]}} - ) # mutable-ok: prisma where filter must be a dict + """End users reset on the budget link like every other gated table, plus a + NULL-budget_id branch: rows created implicitly persist no link and ride the + default tier (litellm.max_end_user_budget_id). + + Matching on the link rather than enumerating user ids keeps a statement's + bind count proportional to the expiring tiers instead of the customer + population, which past ~32,700 dependents exceeds PostgreSQL's per-statement + bind ceiling and wedges the cascade permanently (#40564). + """ + _queue_budget_linked_resets(writes, cascade, extra=_SPENT_ROWS_WHERE) + default_budget_id: Final = litellm.max_end_user_budget_id + if default_budget_id is None or default_budget_id not in cascade.budget_ids: return - tiered: Final = tuple((row.budget_id or litellm.max_end_user_budget_id, row.user_id) for row in cascade.endusers) - for budget_id, cap in cascade.rollover_caps.items(): - if not ( - user_ids := [uid for bid, uid in tiered if bid == budget_id] - ): # mutable-ok: prisma "in" filter takes a list - continue + cap: Final = cascade.rollover_caps.get(default_budget_id) + if cap is None: writes.queue_spend_zero( - where={"user_id": {"in": user_ids}, "spend": {"lte": cap}} + where={"budget_id": None, **_SPENT_ROWS_WHERE} ) # mutable-ok: prisma where filter must be a dict - writes.queue_spend_decrement( - where={"user_id": {"in": user_ids}, "spend": {"gt": cap}}, amount=cap - ) # mutable-ok: prisma where filter must be a dict - plain: Final = [ - uid for bid, uid in tiered if bid is None or bid not in cascade.rollover_caps - ] # mutable-ok: prisma "in" filter takes a list - if plain: - writes.queue_spend_zero(where={"user_id": {"in": plain}}) # mutable-ok: prisma where filter must be a dict + return + writes.queue_spend_zero( + where={"budget_id": None, "spend": {"gt": 0, "lte": cap}} + ) # mutable-ok: prisma where filter must be a dict + writes.queue_spend_decrement( + where={"budget_id": None, "spend": {"gt": cap}}, amount=cap + ) # mutable-ok: prisma where filter must be a dict @dataclass(frozen=True, slots=True) diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index 9f6e1f4c3f7..4103536950d 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -401,7 +401,7 @@ async def test_reset_budget_endusers_are_zeroed_with_the_budget_window_advance() enduser_writes = [c for c in batch_calls if c["table"] == "enduser"] assert len(enduser_writes) == 1 - assert enduser_writes[0]["where"]["user_id"]["in"] == [f"user{i}" for i in range(1, 7)] + assert enduser_writes[0]["where"] == {"budget_id": {"in": ["budget1"]}, "spend": {"gt": 0}} assert enduser_writes[0]["data"] == {"spend": 0} budget_writes = [c for c in batch_calls if c["table"] == "budget"] @@ -602,7 +602,7 @@ async def test_reset_budget_continues_other_categories_on_failure(): assert len([c for c in batch_calls if c["table"] == "team_membership"]) == 1 enduser_writes = [c for c in batch_calls if c["table"] == "enduser"] assert len(enduser_writes) == 1 - assert enduser_writes[0]["where"] == {"user_id": {"in": ["user1"]}} + assert enduser_writes[0]["where"] == {"budget_id": {"in": ["budget1"]}, "spend": {"gt": 0}} assert enduser_writes[0]["data"] == {"spend": 0} # Check the new batch write path: 2 keys + 1 user (user1 failed) + 2 teams. @@ -1031,7 +1031,7 @@ async def test_service_logger_endusers_success(): enduser_writes = [c for c in batch_calls if c["table"] == "enduser"] assert len(enduser_writes) == 1 - assert enduser_writes[0]["where"] == {"user_id": {"in": ["user1", "user2"]}} + assert enduser_writes[0]["where"] == {"budget_id": {"in": ["budget1"]}, "spend": {"gt": 0}} proxy_logging_obj.service_logging_obj.async_service_success_hook.assert_called_once() ( diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 56c0efb41d2..560953f0b51 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -403,7 +403,7 @@ def test_reset_budget_for_enduser(reset_budget_job, mock_prisma_client): { "table": "enduser", "op": "update_many", - "where": {"user_id": {"in": ["test-enduser-1"]}}, + "where": {"budget_id": {"in": ["test-budget-1"]}, "spend": {"gt": 0}}, "data": {"spend": 0}, } ] @@ -504,7 +504,7 @@ def test_reset_budget_all(reset_budget_job, mock_prisma_client): { "table": "enduser", "op": "update_many", - "where": {"user_id": {"in": ["test-enduser-1"]}}, + "where": {"budget_id": {"in": ["test-budget-1"]}, "spend": {"gt": 0}}, "data": {"spend": 0}, } ] @@ -524,6 +524,7 @@ _LINKED_TABLE_CASES = [ ("org", {"budget_id": {"in": ["7d-budget-tier"]}, "spend": {"gt": 0}}), ("tag", {"budget_id": {"in": ["7d-budget-tier"]}, "spend": {"gt": 0}}), ("model_access_group", {"budget_id": {"in": ["7d-budget-tier"]}, "spend": {"gt": 0}}), + ("enduser", {"budget_id": {"in": ["7d-budget-tier"]}, "spend": {"gt": 0}}), ] @@ -553,6 +554,48 @@ def test_budget_table_reset_zeroes_spend_on_every_linked_table( assert writes[0]["data"] == {"spend": 0} +_POSTGRES_MAX_BIND_VARIABLES: Final = 32767 + + +def _bind_count(where: Dict[str, Any]) -> int: + """Bind variables one prisma where-clause compiles to: each scalar is one + placeholder and an ``in`` list contributes one per element.""" + return sum(len(value["in"]) if isinstance(value, dict) and "in" in value else 1 for value in where.values()) + + +@pytest.mark.parametrize("population", [3, 40_000], ids=["small", "over-pg-bind-ceiling"]) +def test_enduser_reset_bind_count_does_not_scale_with_population(reset_budget_job, mock_prisma_client, population): + """Regression for #40564. + + Enumerating every dependent user id put one bind variable per customer into + a single prepared statement. Past PostgreSQL's ceiling the statement could + not be parsed at all, so the whole atomic cascade rolled back, + budget_reset_at never advanced, and the tier stayed due on every later tick + forever. Matching on the budget link keeps the statement the same size no + matter how many customers share a tier. + """ + budget = _budget_row(budget_id="shared-tier", budget_duration="1d") + mock_prisma_client.data["budget"] = [budget] + mock_prisma_client.data["enduser"] = [ + types.SimpleNamespace( + spend=1.0, + litellm_budget_table=budget, + user_id=f"cust-{index:08d}", + budget_id="shared-tier", + ) + for index in range(population) + ] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + writes = _batch_writes(mock_prisma_client, "enduser") + assert [_bind_count(write["where"]) for write in writes] == [2], ( + f"the cascade must not enumerate {population} user ids: past " + f"{_POSTGRES_MAX_BIND_VARIABLES} binds PostgreSQL refuses the statement, got {writes[:1]}" + ) + assert _batch_writes(mock_prisma_client, "budget")[0]["data"]["budget_reset_at"] is not None + + def test_budget_table_reset_writes_nothing_when_no_budget_is_due(reset_budget_job, mock_prisma_client): """Nothing due means no transaction is opened at all.""" asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) @@ -720,14 +763,22 @@ def test_reset_budget_resets_endusers_with_null_budget_id(reset_budget_job, mock asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) - # Both end users are zeroed by the same committed statement. - enduser_writes = _batch_writes(mock_prisma_client, "enduser") - assert len(enduser_writes) == 1, f"Expected a single enduser write, got {enduser_writes}" - assert set(enduser_writes[0]["where"]["user_id"]["in"]) == { - "enduser-explicit", - "enduser-implicit", - } - assert enduser_writes[0]["data"] == {"spend": 0} + # Both end users are zeroed: the linked rows on the tier's budget_id, the + # implicit ones on the NULL branch that stands in for the default tier. + assert _batch_writes(mock_prisma_client, "enduser") == [ + { + "table": "enduser", + "op": "update_many", + "where": {"budget_id": {"in": [default_budget_id]}, "spend": {"gt": 0}}, + "data": {"spend": 0}, + }, + { + "table": "enduser", + "op": "update_many", + "where": {"budget_id": None, "spend": {"gt": 0}}, + "data": {"spend": 0}, + }, + ] # Verify find_many was called to fetch NULL-budget-id end users find_many_calls = mock_prisma_client.db.litellm_endusertable.find_many_calls @@ -3043,13 +3094,13 @@ def test_budget_cascade_carries_enduser_overage_when_rollover_enabled( assert { "table": "enduser", "op": "update_many", - "where": {"user_id": {"in": ["enduser-roll"]}, "spend": {"gt": 10.0}}, + "where": {"budget_id": "budget-roll", "spend": {"gt": 10.0}}, "data": {"spend": {"decrement": 10.0}}, } in enduser_writes assert { "table": "enduser", "op": "update_many", - "where": {"user_id": {"in": ["enduser-roll"]}, "spend": {"lte": 10.0}}, + "where": {"budget_id": "budget-roll", "spend": {"gt": 0, "lte": 10.0}}, "data": {"spend": 0}, } in enduser_writes From ae018825357ddafcf0da15d903d10df0f2581c59 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:14:13 -0700 Subject: [PATCH 386/411] feat(proxy): offload spend tracking to a pod-local collector sidecar (#40545) * feat(proxy): offload spend tracking to a pod-local spend worker sidecar py-spy on the gateway showed the post-response _PROXY_track_cost_callback, spend-log and DBSpendUpdateWriter work running on the inference workers' event loop, so a DB or Redis stall backed up the request path. When LITELLM_SPEND_WORKER_ENABLED=true, _ProxyDBLogger serializes one compact typed SpendEvent per success and hands it to a SpendEventProducer that ships it over a unix socket (default) or loopback-only TCP to a sidecar started as `python -m gateway.spend_worker`. The sidecar runs the unchanged _ProxyDBLogger pipeline against the pod's PgBouncer (pooled_database_url). When the sidecar is unreachable, the buffer is full, or the gateway shuts down with events still queued or in flight, the producer applies LITELLM_SPEND_WORKER_ON_UNAVAILABLE (fallback in-process, or drop). The sidecar half-closes producers on SIGTERM and drains, the producer treats EOF as unavailable, and the gateway flushes buffered spend counters on shutdown. The sidecar honors LITELLM_LOG so its writes are visible in its own process log. Helm: both charts gain an opt-in spend-worker sidecar container sharing an emptyDir socket dir, and the componentized chart's HPA uses a ContainerResource CPU metric scoped to the gateway container so sidecar CPU does not drive inference scaling. * feat(terraform): opt-in spend-worker sidecar for the AWS and GCP gateway stacks Adds spend_worker_* inputs to both modules. On ECS Fargate the sidecar is a second, non-essential container in the gateway task; on Cloud Run it is a second container in the gateway service. Both listen on loopback TCP, share the gateway's DB/Redis/secret env, and set LITELLM_JOB_ROLE=spend_worker. Disabled by default. Plan-only tests cover both, and the terraform CI workflow now runs the gcp module too Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): retrieve a completed batch in the in-process spend path test The base now defers cost tracking for batches that are still in flight, so an in_progress batch never reaches update_database Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): rename the spend worker sidecar to collector Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): run the collector from the installed litellm package and finish in-flight fallbacks on shutdown The sidecar command becomes python -m litellm.proxy.collector so the classic image, whose runtime stage copies only the installed package, can run it. The module now assembles DATABASE_URL and the pod-local pgbouncer URL itself, replacing gateway/collector.py The componentized collector sidecar inherits gateway.volumeMounts so custom CA mounts reach it. SpendEventProducer shields an in-progress fallback from the writer task cancellation so close() no longer loses an event already handed to the in-process pipeline Helpers used across modules (address_argument, should_store_prompts_and_responses_in_spend_logs, flush_spend_counters_on_shutdown) become public so the change adds no reportPrivateUsage errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(terraform): drop the gcp job duplicated by the aws/gcp matrix Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(collector): keep metrics env off the classic sidecar and reject shared loopback ports The classic chart no longer hands PROMETHEUS_METRICS_PORT and the billing metrics env to the collector container, and gives it the same /.npm scratch mount as the proxy on a read-only root. AWS and GCP now refuse a plan where the spend collector and the metrics sidecar bind the same loopback port. A regression test drives a sidecar crash mid-stream on asyncio and uvloop and checks no event is billed by both the sidecar and the in-process fallback; the producer docstring spells out why a failed drain() cannot double count Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(proxy): format pooled_database_url after the pgbouncer rebase Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep the cache-hit preset key and survive dead producers on collector drain Cache hits updated the logging object after the early return, so the offloaded spend event carried preset_cache_key=None and the collector re-hashed reconstructed kwargs. Also guard write_eof() against producer transports uvloop already closed so one dead connection cannot abort the drain Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(terraform): keep the gcp collector port off the metrics sidecar health port Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): collector connects to Postgres directly under IAM or Entra token auth Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): mark the collector's DATABASE_URL as pooled when it uses the pod's pgbouncer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-terraform-modules.yml | 40 +- helm/litellm-helm/templates/_helpers.tpl | 160 +++++++ helm/litellm-helm/templates/deployment.yaml | 180 +++----- helm/litellm-helm/templates/hpa.yaml | 10 + helm/litellm-helm/tests/collector_tests.yaml | 272 ++++++++++++ helm/litellm-helm/values.yaml | 42 ++ helm/litellm/templates/_helpers.tpl | 31 ++ .../litellm/templates/gateway/deployment.yaml | 56 ++- helm/litellm/templates/gateway/hpa.yaml | 10 + helm/litellm/tests/collector_tests.yaml | 204 +++++++++ helm/litellm/values.yaml | 36 ++ litellm/caching/caching_handler.py | 4 +- litellm/proxy/collector.py | 220 +++++++++ litellm/proxy/db/pgbouncer.py | 16 + .../proxy/hooks/proxy_track_cost_callback.py | 54 ++- litellm/proxy/proxy_server.py | 39 +- litellm/proxy/spend_tracking/spend_event.py | 418 ++++++++++++++++++ .../spend_tracking/spend_event_producer.py | 338 ++++++++++++++ .../spend_tracking/spend_tracking_utils.py | 24 +- litellm/utils.py | 3 + terraform/litellm/README.md | 1 + terraform/litellm/aws/README.md | 33 ++ terraform/litellm/aws/ecs.tf | 82 +++- .../litellm/aws/tests/collector.tftest.hcl | 144 ++++++ terraform/litellm/aws/variables.tf | 70 +++ terraform/litellm/gcp/README.md | 37 ++ terraform/litellm/gcp/cloudrun.tf | 85 +++- .../litellm/gcp/tests/collector.tftest.hcl | 165 +++++++ terraform/litellm/gcp/variables.tf | 70 +++ tests/test_gateway/test_launch.py | 6 +- .../caching/test_caching_handler.py | 35 ++ .../hooks/test_proxy_track_cost_callback.py | 255 ++++++++++- .../proxy/proxy_server/test_lifecycle.py | 32 ++ .../proxy/spend_tracking/test_spend_event.py | 213 +++++++++ .../test_spend_event_producer.py | 359 +++++++++++++++ .../test_spend_tracking_utils.py | 122 +++-- tests/test_litellm/proxy/test_collector.py | 229 ++++++++++ 37 files changed, 3865 insertions(+), 230 deletions(-) create mode 100644 helm/litellm-helm/tests/collector_tests.yaml create mode 100644 helm/litellm/tests/collector_tests.yaml create mode 100644 litellm/proxy/collector.py create mode 100644 litellm/proxy/spend_tracking/spend_event.py create mode 100644 litellm/proxy/spend_tracking/spend_event_producer.py create mode 100644 terraform/litellm/aws/tests/collector.tftest.hcl create mode 100644 terraform/litellm/gcp/tests/collector.tftest.hcl create mode 100644 tests/test_litellm/proxy/spend_tracking/test_spend_event.py create mode 100644 tests/test_litellm/proxy/spend_tracking/test_spend_event_producer.py create mode 100644 tests/test_litellm/proxy/test_collector.py diff --git a/.github/workflows/test-terraform-modules.yml b/.github/workflows/test-terraform-modules.yml index 52006d9b578..e6896604b7f 100644 --- a/.github/workflows/test-terraform-modules.yml +++ b/.github/workflows/test-terraform-modules.yml @@ -25,13 +25,17 @@ concurrency: cancel-in-progress: true jobs: - aws-module: - name: fmt, validate, test (aws) + module: + name: fmt, validate, test (${{ matrix.module }}) runs-on: ubuntu-latest timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + module: [aws, gcp] defaults: run: - working-directory: terraform/litellm/aws + working-directory: terraform/litellm/${{ matrix.module }} steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: @@ -51,35 +55,7 @@ jobs: - name: validate run: terraform validate - # Plan-only, mock_provider-backed: no AWS credentials, no API calls. + # Plan-only, mock_provider-backed: no cloud credentials, no API calls. - name: test run: terraform test - gcp-module: - name: fmt, validate, test (gcp) - runs-on: ubuntu-latest - timeout-minutes: 15 - defaults: - run: - working-directory: terraform/litellm/gcp - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 - with: - terraform_version: 1.13.3 - terraform_wrapper: false - - - name: fmt - run: terraform fmt -recursive -check -diff - - - name: init - run: terraform init -backend=false -input=false - - - name: validate - run: terraform validate - - - name: test - run: terraform test diff --git a/helm/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl index 8f2acb20fce..9630633912e 100644 --- a/helm/litellm-helm/templates/_helpers.tpl +++ b/helm/litellm-helm/templates/_helpers.tpl @@ -161,3 +161,163 @@ taken before the change, which by that point no longer exists. {{- fail (printf "postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got %q). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore." $tag) -}} {{- end -}} {{- end -}} + +{{/* +Environment shared by the proxy container and the opt-in collector sidecar: +database, pgbouncer, master key, redis, user envVars. Both containers must see +the same DATABASE_URL and REDIS_* so the sidecar reaches the pod's pgbouncer +and the same spend transaction buffer. +*/}} +{{- define "litellm.proxyEnv" -}} +- name: HOST + value: "{{ .Values.listen | default "0.0.0.0" }}" +- name: PORT + value: {{ .Values.service.port | quote}} +{{- if .Values.db.deployStandalone }} +- name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "litellm.fullname" . }}-dbcredentials + key: username +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "litellm.fullname" . }}-dbcredentials + key: password +- name: DATABASE_HOST + value: {{ .Release.Name }}-postgresql +- name: DATABASE_NAME + value: litellm +{{- else if .Values.db.useExisting }} +- name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.usernameKey }} +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.passwordKey }} +- name: DATABASE_HOST + {{- if .Values.db.secret.endpointKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.endpointKey }} + {{- else }} + value: {{ .Values.db.endpoint }} + {{- end }} +- name: DATABASE_NAME + value: {{ .Values.db.database }} +- name: DATABASE_URL + value: {{ .Values.db.url | quote }} +{{- end }} +{{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }} +- name: DATABASE_READER_HOST + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.readReplicaEndpointKey }} +{{- end }} +{{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }} +- name: DATABASE_URL_READ_REPLICA + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.readReplicaUrlKey }} +{{- else if .Values.db.readReplicaUrl }} +- name: DATABASE_URL_READ_REPLICA + value: {{ .Values.db.readReplicaUrl | quote }} +{{- end }} +{{- if .Values.db.connectionPool.enabled }} +- name: LITELLM_PGBOUNCER_ENABLED + value: "true" +- name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: {{ .Values.db.connectionPool.maxDbConnections | quote }} +- name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN + value: {{ .Values.db.connectionPool.maxClientConn | quote }} +{{- end }} +- name: PROXY_MASTER_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }} + key: {{ .Values.masterkeySecretKey | default "masterkey" }} +{{- if .Values.redis.enabled }} +- name: REDIS_HOST + value: {{ include "litellm.redis.serviceName" . }} +- name: REDIS_PORT + value: {{ include "litellm.redis.port" . | quote }} +- name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "redis.secretName" .Subcharts.redis }} + key: {{include "redis.secretPasswordKey" .Subcharts.redis }} +{{- end }} +{{- /* + Inject LITELLM_LOG only when envVars does not already define it. +*/}} +{{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }} +- name: LITELLM_LOG + value: {{ .Values.logLevel | quote }} +{{- end }} +{{- if .Values.envVars }} +{{- range $key, $val := .Values.envVars }} +- name: {{ $key }} + value: {{ $val | quote }} +{{- end }} +{{- end }} +{{- with .Values.extraEnvVars }} +{{ toYaml . }} +{{- end }} +{{- if .Values.migrationJob.enabled }} +# Schema updates are owned by the dedicated migrations Job; skip +# the proxy's startup `prisma db push` so N replicas don't race +# one DB on every rollout. Placed last (after envVars and +# extraEnvVars) so this override can't be silently shadowed by a +# user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env +# semantics — same pattern the migrations Job uses. +- name: DISABLE_SCHEMA_UPDATE + value: "true" +{{- end }} +{{- end -}} + +{{/* +Proxy-only metering and metrics env. The collector sidecar serves no HTTP +traffic, so it gets neither. +*/}} +{{- define "litellm.proxyMetricsEnv" -}} +{{- if .Values.billingMetrics.enabled }} +{{ include "litellm.billingMetricsEnv" . }} +{{- end }} +{{- if .Values.metricsServer.enabled }} +{{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }} +{{- fail "metricsServer.port must differ from service.port" }} +{{- end }} +- name: PROMETHEUS_METRICS_PORT + value: {{ .Values.metricsServer.port | quote }} +{{- end }} +{{- end -}} + +{{/* +Directory of the collector's unix socket, shared between the two containers +through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP. +*/}} +{{- define "litellm.collector.socketDir" -}} +{{- if and .Values.collector.enabled (hasPrefix "unix://" .Values.collector.address) -}} +{{- dir (trimPrefix "unix://" .Values.collector.address) -}} +{{- end -}} +{{- end -}} + +{{- define "litellm.collectorEnv" -}} +- name: LITELLM_COLLECTOR_ENABLED + value: "true" +- name: LITELLM_COLLECTOR_ADDRESS + value: {{ .Values.collector.address | quote }} +- name: LITELLM_COLLECTOR_BUFFER_SIZE + value: {{ .Values.collector.bufferSize | quote }} +- name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: {{ .Values.collector.onUnavailable | quote }} +- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS + value: {{ .Values.collector.drainTimeoutSeconds | quote }} +{{- end -}} diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml index 834071eb9b2..cf7b3f8a38d 100644 --- a/helm/litellm-helm/templates/deployment.yaml +++ b/helm/litellm-helm/templates/deployment.yaml @@ -56,126 +56,10 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: HOST - value: "{{ .Values.listen | default "0.0.0.0" }}" - - name: PORT - value: {{ .Values.service.port | quote}} - {{- if .Values.db.deployStandalone }} - - name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: username - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: password - - name: DATABASE_HOST - value: {{ .Release.Name }}-postgresql - - name: DATABASE_NAME - value: litellm - {{- else if .Values.db.useExisting }} - - name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.usernameKey }} - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.passwordKey }} - - name: DATABASE_HOST - {{- if .Values.db.secret.endpointKey }} - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.endpointKey }} - {{- else }} - value: {{ .Values.db.endpoint }} - {{- end }} - - name: DATABASE_NAME - value: {{ .Values.db.database }} - - name: DATABASE_URL - value: {{ .Values.db.url | quote }} - {{- end }} - {{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }} - - name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.readReplicaEndpointKey }} - {{- end }} - {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }} - - name: DATABASE_URL_READ_REPLICA - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.readReplicaUrlKey }} - {{- else if .Values.db.readReplicaUrl }} - - name: DATABASE_URL_READ_REPLICA - value: {{ .Values.db.readReplicaUrl | quote }} - {{- end }} - {{- if .Values.db.connectionPool.enabled }} - - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: {{ .Values.db.connectionPool.maxDbConnections | quote }} - - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: {{ .Values.db.connectionPool.maxClientConn | quote }} - {{- end }} - - name: PROXY_MASTER_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }} - key: {{ .Values.masterkeySecretKey | default "masterkey" }} - {{- if .Values.redis.enabled }} - - name: REDIS_HOST - value: {{ include "litellm.redis.serviceName" . }} - - name: REDIS_PORT - value: {{ include "litellm.redis.port" . | quote }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "redis.secretName" .Subcharts.redis }} - key: {{include "redis.secretPasswordKey" .Subcharts.redis }} - {{- end }} - {{- /* - Inject LITELLM_LOG only when envVars does not already define it. - */}} - {{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }} - - name: LITELLM_LOG - value: {{ .Values.logLevel | quote }} - {{- end }} - {{- if .Values.envVars }} - {{- range $key, $val := .Values.envVars }} - - name: {{ $key }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with .Values.extraEnvVars }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsEnv" . | nindent 12 }} - {{- end }} - {{- if .Values.metricsServer.enabled }} - {{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }} - {{- fail "metricsServer.port must differ from service.port" }} - {{- end }} - - name: PROMETHEUS_METRICS_PORT - value: {{ .Values.metricsServer.port | quote }} - {{- end }} - {{- if .Values.migrationJob.enabled }} - # Schema updates are owned by the dedicated migrations Job; skip - # the proxy's startup `prisma db push` so N replicas don't race - # one DB on every rollout. Placed last (after envVars and - # extraEnvVars) so this override can't be silently shadowed by a - # user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env - # semantics — same pattern the migrations Job uses. - - name: DISABLE_SCHEMA_UPDATE - value: "true" + {{- include "litellm.proxyEnv" . | nindent 12 }} + {{- include "litellm.proxyMetricsEnv" . | nindent 12 }} + {{- if .Values.collector.enabled }} + {{- include "litellm.collectorEnv" . | nindent 12 }} {{- end }} envFrom: {{- range .Values.environmentSecrets }} @@ -253,6 +137,10 @@ spec: {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} {{- end }} + {{- if include "litellm.collector.socketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.collector.socketDir" . }} + {{- end }} {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -260,6 +148,53 @@ spec: lifecycle: {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.collector.enabled }} + - name: {{ include "litellm.name" . }}-collector + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: {{ toYaml .Values.collector.command | nindent 12 }} + env: + {{- include "litellm.proxyEnv" . | nindent 12 }} + {{- include "litellm.collectorEnv" . | nindent 12 }} + - name: LITELLM_JOB_ROLE + value: collector + {{- if not (hasKey (default dict .Values.envVars) "CONFIG_FILE_PATH") }} + - name: CONFIG_FILE_PATH + value: /etc/litellm/config.yaml + {{- end }} + envFrom: + {{- range .Values.environmentSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- range .Values.environmentConfigMaps }} + - configMapRef: + name: {{ . }} + {{- end }} + resources: + {{- toYaml .Values.collector.resources | nindent 12 }} + volumeMounts: + - name: litellm-config + mountPath: /etc/litellm/config.yaml + subPath: config.yaml + {{- if include "litellm.collector.socketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.collector.socketDir" . }} + {{- end }} + {{ if .Values.securityContext.readOnlyRootFilesystem }} + - name: tmp + mountPath: /tmp + - name: cache + mountPath: /.cache + - name: npm + mountPath: /.npm + {{- end }} + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} {{- with .Values.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} @@ -288,6 +223,11 @@ spec: {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} {{- end }} + {{- if include "litellm.collector.socketDir" . }} + - name: collector-socket + emptyDir: + sizeLimit: 1Mi + {{- end }} {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/litellm-helm/templates/hpa.yaml b/helm/litellm-helm/templates/hpa.yaml index 010c4095ab3..a651f916d21 100644 --- a/helm/litellm-helm/templates/hpa.yaml +++ b/helm/litellm-helm/templates/hpa.yaml @@ -18,6 +18,15 @@ spec: {{- end }} metrics: {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- if and .Values.collector.enabled .Values.collector.scaleOnProxyContainerCpu }} + - type: ContainerResource + containerResource: + name: cpu + container: {{ include "litellm.name" . }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- else }} - type: Resource resource: name: cpu @@ -25,6 +34,7 @@ spec: type: Utilization averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} {{- end }} + {{- end }} {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: diff --git a/helm/litellm-helm/tests/collector_tests.yaml b/helm/litellm-helm/tests/collector_tests.yaml new file mode 100644 index 00000000000..0340b1161b7 --- /dev/null +++ b/helm/litellm-helm/tests/collector_tests.yaml @@ -0,0 +1,272 @@ +suite: test collector sidecar +templates: + - deployment.yaml + - hpa.yaml + - configmap-litellm.yaml +tests: + - it: should run the proxy alone with no collector env by default + template: deployment.yaml + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ENABLED + value: "true" + - notContains: + path: spec.template.spec.volumes + content: + name: collector-socket + any: true + + - it: should add the sidecar on the same image and point both containers at the unix socket + template: deployment.yaml + set: + image.tag: test + db.connectionPool.enabled: true + collector.enabled: true + collector.resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 2 + - equal: + path: spec.template.spec.containers[1].name + value: litellm-collector + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm:test + - equal: + path: spec.template.spec.containers[1].command + value: [python, -m, litellm.proxy.collector] + - equal: + path: spec.template.spec.containers[1].resources.requests.cpu + value: 500m + - equal: + path: spec.template.spec.containers[1].resources.limits.memory + value: 2Gi + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: unix:///var/run/litellm/collector.sock + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_BUFFER_SIZE + value: "1000" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: fallback + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_JOB_ROLE + value: collector + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_JOB_ROLE + value: collector + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: unix:///var/run/litellm/collector.sock + - contains: + path: spec.template.spec.containers[1].env + content: + name: CONFIG_FILE_PATH + value: /etc/litellm/config.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: DATABASE_HOST + value: RELEASE-NAME-postgresql + - contains: + path: spec.template.spec.containers[1].env + content: + name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: RELEASE-NAME-litellm-dbcredentials + key: password + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS + value: "20" + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: collector-socket + mountPath: /var/run/litellm + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: collector-socket + mountPath: /var/run/litellm + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: litellm-config + mountPath: /etc/litellm/config.yaml + subPath: config.yaml + - contains: + path: spec.template.spec.volumes + content: + name: collector-socket + emptyDir: + sizeLimit: 1Mi + + - it: should skip the socket volume and pass the policy through on tcp transport + template: deployment.yaml + set: + collector.enabled: true + collector.address: tcp://127.0.0.1:4100 + collector.onUnavailable: drop + collector.bufferSize: 50 + envVars: + CONFIG_FILE_PATH: /custom/config.yaml + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 2 + - notContains: + path: spec.template.spec.containers[1].env + content: + name: CONFIG_FILE_PATH + value: /etc/litellm/config.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: CONFIG_FILE_PATH + value: /custom/config.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: tcp://127.0.0.1:4100 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: drop + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_BUFFER_SIZE + value: "50" + - notContains: + path: spec.template.spec.volumes + content: + name: collector-socket + any: true + + - it: should keep metrics and billing env on the proxy container only + template: deployment.yaml + set: + collector.enabled: true + metricsServer.enabled: true + metricsServer.port: 9090 + billingMetrics.enabled: true + billingMetrics.endpoint: https://metering.example.com + billingMetrics.secretName: billing-mtls + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROMETHEUS_METRICS_PORT + value: "9090" + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_BILLING_METRICS_ENDPOINT + value: https://metering.example.com + - notContains: + path: spec.template.spec.containers[1].env + content: + name: PROMETHEUS_METRICS_PORT + any: true + - notContains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_BILLING_METRICS_ENDPOINT + any: true + - notContains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: billing-metrics-mtls + any: true + + - it: should give the sidecar the same scratch mounts as the proxy on a read-only root + template: deployment.yaml + set: + collector.enabled: true + securityContext.readOnlyRootFilesystem: true + asserts: + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: npm + mountPath: /.npm + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: cache + mountPath: /.cache + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: tmp + mountPath: /tmp + + - it: should keep the pod-wide cpu metric unless asked to scale on the proxy container + template: hpa.yaml + set: + autoscaling.enabled: true + collector.enabled: true + asserts: + - equal: { path: "spec.metrics[0].type", value: Resource } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + + - it: should scale on the proxy container's cpu only when opted in + template: hpa.yaml + set: + autoscaling.enabled: true + collector.enabled: true + collector.scaleOnProxyContainerCpu: true + asserts: + - equal: { path: "spec.metrics[0].type", value: ContainerResource } + - equal: { path: "spec.metrics[0].containerResource.name", value: cpu } + - equal: { path: "spec.metrics[0].containerResource.container", value: litellm } + - equal: { path: "spec.metrics[0].containerResource.target.averageUtilization", value: 60 } + - isNull: { path: "spec.metrics[0].resource" } + + - it: should not switch to the container metric while the sidecar is off + template: hpa.yaml + set: + autoscaling.enabled: true + collector.scaleOnProxyContainerCpu: true + asserts: + - equal: { path: "spec.metrics[0].type", value: Resource } diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index db596e2d68e..fcee331a5aa 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -190,6 +190,48 @@ metricsServer: enabled: false port: 4001 +# Opt-in sidecar that runs the post-response spend pipeline (cost calculation, +# spend logs, spend counters, budget reservation reconciliation) so the proxy's +# uvicorn workers only serialise a compact typed event and go back to serving +# inference. Same image and tag as the proxy, second container in the same pod, +# fed over loopback (a unix socket on a shared emptyDir, or 127.0.0.1 TCP). It +# reuses the pod's in-container pgbouncer (db.connectionPool) and the same Redis +# spend transaction buffer, so the per-pod DB connection budget is unchanged. +# Delivery is at-most-once inside the pod: events already handed to the sidecar +# are lost if it crashes before writing them; events the workers could not hand +# over follow onUnavailable. Both containers drain on SIGTERM within +# terminationGracePeriodSeconds +collector: + enabled: false + # unix:////.sock (the becomes a shared emptyDir) or tcp://127.0.0.1: + address: unix:///var/run/litellm/collector.sock + # Events each uvicorn worker holds in memory while the sidecar is slow or restarting + bufferSize: 1000 + # fallback: run the pipeline in the worker when the sidecar is unreachable or the + # buffer is full (spend stays exact, that request costs proxy CPU again) + # drop: count and discard the event instead (spend under-reports) + onUnavailable: fallback + # How long the workers keep pushing buffered events on shutdown, and how long the + # sidecar keeps serving its open connections after SIGTERM + drainTimeoutSeconds: 10 + command: + - python + - -m + - litellm.proxy.collector + # Sized independently of the proxy container; the pipeline is CPU bound + resources: {} + # requests: + # cpu: 500m + # memory: 1Gi + # limits: + # cpu: "1" + # memory: 2Gi + # When autoscaling.enabled, swap the pod-wide cpu Resource metric for an + # autoscaling/v2 ContainerResource metric on the proxy container only, so the + # sidecar's CPU never scales inference replicas. Needs Kubernetes 1.30+ (or the + # HPAContainerMetrics feature gate on 1.27 to 1.29) + scaleOnProxyContainerCpu: false + resources: {} # Unset by default so the chart installs on small clusters such as Minikube, and so an diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index dc47bc0b0a1..692a799e783 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -457,3 +457,34 @@ ImplementationSpecific {{- end -}} {{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}} + +{{/* +Directory of the collector's unix socket, shared by the gateway and +collector containers through an emptyDir. Empty when the sidecar is off +or gateway.collector.address is a tcp://127.0.0.1: address. +*/}} +{{- define "litellm.gateway.collectorSocketDir" -}} +{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}} +{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}} +{{- end -}} +{{- end -}} + +{{/* +LITELLM_COLLECTOR_* env shared by the producer (gateway container) and the +consumer (collector container), so both agree on the transport and the +shutdown drain window. +*/}} +{{- define "litellm.gateway.collectorEnv" -}} +{{- with .Values.gateway.collector }} +- name: LITELLM_COLLECTOR_ENABLED + value: "true" +- name: LITELLM_COLLECTOR_ADDRESS + value: {{ .address | quote }} +- name: LITELLM_COLLECTOR_BUFFER_SIZE + value: {{ .bufferSize | quote }} +- name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: {{ .onUnavailable | quote }} +- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS + value: {{ .drainTimeoutSeconds | quote }} +{{- end }} +{{- end -}} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 1e9041f0a33..5c1a089b50a 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -74,8 +74,11 @@ spec: - name: PROMETHEUS_MULTIPROC_DIR value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} {{- end }} + {{- if .Values.gateway.collector.enabled }} + {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} + {{- end }} {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} volumeMounts: {{- if .Values.gateway.config.create }} - name: gateway-config @@ -86,6 +89,10 @@ spec: - name: prometheus-multiproc mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} {{- end }} @@ -145,10 +152,50 @@ spec: resources: {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }} {{- end }} + {{- if .Values.gateway.collector.enabled }} + - name: collector + image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: + - python + - -m + - litellm.proxy.collector + env: + {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} + {{- if .Values.gateway.config.create }} + - name: CONFIG_FILE_PATH + value: /app/config/config.yaml + {{- end }} + {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} + - name: LITELLM_JOB_ROLE + value: collector + {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts (include "litellm.gateway.collectorSocketDir" .) }} + volumeMounts: + {{- if .Values.gateway.config.create }} + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} + {{- end }} + {{- with .Values.gateway.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + resources: + {{- toYaml .Values.gateway.collector.resources | nindent 12 }} + {{- end }} {{- with .Values.gateway.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }} + {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} volumes: {{- if .Values.gateway.config.create }} - name: gateway-config @@ -159,6 +206,11 @@ spec: - name: prometheus-multiproc emptyDir: {} {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + emptyDir: + sizeLimit: 1Mi + {{- end }} {{- if .Values.billingMetrics.enabled }} {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} {{- end }} diff --git a/helm/litellm/templates/gateway/hpa.yaml b/helm/litellm/templates/gateway/hpa.yaml index 4183079cd31..e7094e96106 100644 --- a/helm/litellm/templates/gateway/hpa.yaml +++ b/helm/litellm/templates/gateway/hpa.yaml @@ -15,6 +15,15 @@ spec: maxReplicas: {{ .Values.gateway.hpa.maxReplicas }} metrics: {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- if and .Values.gateway.collector.enabled .Values.gateway.collector.scaleOnGatewayContainerCpu }} + - type: ContainerResource + containerResource: + name: cpu + container: gateway + target: + type: Utilization + averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- else }} - type: Resource resource: name: cpu @@ -22,6 +31,7 @@ spec: type: Utilization averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} {{- end }} + {{- end }} {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }} - type: Resource resource: diff --git a/helm/litellm/tests/collector_tests.yaml b/helm/litellm/tests/collector_tests.yaml new file mode 100644 index 00000000000..b1199a16793 --- /dev/null +++ b/helm/litellm/tests/collector_tests.yaml @@ -0,0 +1,204 @@ +suite: test gateway collector sidecar +templates: + - gateway/configmap.yaml + - gateway/deployment.yaml + - gateway/hpa.yaml +values: + - ./values/required.yaml +tests: + - it: adds no sidecar, env, volume or container metric when the collector is off + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ENABLED + value: "true" + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.volumes + content: + name: collector-socket + any: true + template: gateway/deployment.yaml + - equal: + path: spec.metrics[0].type + value: Resource + template: gateway/hpa.yaml + + - it: runs the collector as a sidecar sharing env, config and a unix socket emptyDir, and scales on the gateway container only + set: + gateway.collector.enabled: true + gateway.collector.bufferSize: 250 + gateway.collector.onUnavailable: drop + gateway.image.tag: v1.102.0 + gateway.numWorkers: 4 + gateway.extraEnv: + - name: LITELLM_PGBOUNCER_ENABLED + value: "true" + gateway.envSecrets: + - litellm-license + gateway.volumes: + - name: redis-ca + secret: + secretName: redis-ca + gateway.volumeMounts: + - name: redis-ca + mountPath: /etc/litellm/redis-ca + readOnly: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: unix:///var/run/litellm/collector.sock + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_BUFFER_SIZE + value: "250" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ON_UNAVAILABLE + value: drop + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: collector-socket + mountPath: /var/run/litellm + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].name + value: collector + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm-gateway:v1.102.0 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].command + value: + - python + - -m + - litellm.proxy.collector + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_JOB_ROLE + value: collector + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: CONFIG_FILE_PATH + value: /app/config/config.yaml + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_PGBOUNCER_ENABLED + value: "true" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: DATABASE_HOST + value: postgres.example.com + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: unix:///var/run/litellm/collector.sock + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[1].env + content: + name: NUM_WORKERS + any: true + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].envFrom + value: + - secretRef: + name: litellm-license + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: collector-socket + mountPath: /var/run/litellm + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: redis-ca + mountPath: /etc/litellm/redis-ca + readOnly: true + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].resources.limits.cpu + value: "1" + template: gateway/deployment.yaml + - contains: + path: spec.template.spec.volumes + content: + name: collector-socket + emptyDir: + sizeLimit: 1Mi + template: gateway/deployment.yaml + - equal: + path: spec.metrics[0] + value: + type: ContainerResource + containerResource: + name: cpu + container: gateway + target: + type: Utilization + averageUtilization: 70 + template: gateway/hpa.yaml + + - it: uses loopback tcp without a socket volume and keeps the pod-wide cpu metric when asked + set: + gateway.collector.enabled: true + gateway.collector.address: tcp://127.0.0.1:4010 + gateway.collector.scaleOnGatewayContainerCpu: false + asserts: + - contains: + path: spec.template.spec.containers[1].env + content: + name: LITELLM_COLLECTOR_ADDRESS + value: tcp://127.0.0.1:4010 + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.volumes + content: + name: collector-socket + any: true + template: gateway/deployment.yaml + - notContains: + path: spec.template.spec.containers[1].volumeMounts + content: + name: collector-socket + any: true + template: gateway/deployment.yaml + - equal: + path: spec.metrics[0].type + value: Resource + template: gateway/hpa.yaml diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 677e21a5a51..1f67b984513 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -314,6 +314,42 @@ gateway: labels: {} interval: 15s scrapeTimeout: 10s + # Opt-in `collector` sidecar (same image, `python -m litellm.proxy.collector`) + # that runs the post-response spend pipeline (cost calculation, spend logs, + # spend counters, budget reservation reconciliation) so the uvicorn workers + # only serialise a compact event over loopback and go back to serving + # requests. It shares the pod's env, proxy config, in-container pgbouncer and + # Redis spend buffer, so the per-pod DB connection budget is unchanged. + # Delivery is at-most-once inside the pod: events already handed over are + # lost if the sidecar dies before writing them; events the workers cannot + # hand over follow `onUnavailable`. + collector: + enabled: false + # unix:////.sock (the becomes a shared emptyDir) or + # tcp://127.0.0.1: + address: unix:///var/run/litellm/collector.sock + # Events each uvicorn worker holds in memory while the sidecar is slow or + # restarting. + bufferSize: 1000 + # fallback: run the pipeline in the worker when the sidecar is unreachable + # or the buffer is full (spend stays exact, that request costs gateway CPU + # again). drop: count and discard the event instead (spend under-reports). + onUnavailable: fallback + # How long the workers keep pushing buffered events on shutdown, and how + # long the sidecar keeps serving open connections after SIGTERM. + drainTimeoutSeconds: 10 + # Sized independently of the gateway container; the pipeline is CPU bound. + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + # With hpa.targetCPUUtilizationPercentage set, scale on an autoscaling/v2 + # ContainerResource metric of the `gateway` container only, so the + # sidecar's CPU never drives inference replicas. Needs Kubernetes 1.30+. + scaleOnGatewayContainerCpu: true image: repository: ghcr.io/berriai/litellm-gateway tag: "" # defaults to .Chart.AppVersion diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 0de88eacaa5..139dcf058d2 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -1217,7 +1217,9 @@ class LLMCachingHandler: } if litellm.cache is not None: - litellm_params["preset_cache_key"] = litellm.cache._get_preset_cache_key_from_kwargs(**kwargs) + litellm_params["preset_cache_key"] = ( + self.preset_cache_key or litellm.cache._get_preset_cache_key_from_kwargs(**kwargs) + ) else: litellm_params["preset_cache_key"] = None diff --git a/litellm/proxy/collector.py b/litellm/proxy/collector.py new file mode 100644 index 00000000000..3ff2a83860b --- /dev/null +++ b/litellm/proxy/collector.py @@ -0,0 +1,220 @@ +"""Collector sidecar: consume spend events from the pod's inference workers and run the cost pipeline. + +Runs the proxy startup lifespan (config, Prisma, Redis transaction buffer, scheduled spend flushes) +without serving HTTP, then listens on ``LITELLM_COLLECTOR_ADDRESS`` for newline-delimited spend +events. Each event goes through the unchanged ``_ProxyDBLogger._PROXY_track_cost_callback``, so +spend logs, spend counters, budget reservation reconciliation and cache updates happen exactly as +they would in-process, just in this container. Events are handled in order per producer connection +(one per uvicorn worker); a slow pipeline fills the socket buffer and the producer's bounded queue, +which is the backpressure that triggers its fallback or drop policy. ``SIGTERM`` stops accepting +connections, half-closes every producer connection so the producers switch to their unavailable +policy, finishes the events already sent, then runs the proxy shutdown (which flushes the buffered +spend transactions). + +``DATABASE_URL`` is assembled from the same ``DATABASE_*`` inputs as the proxy container, and when +``LITELLM_PGBOUNCER_ENABLED`` is set it points at the PgBouncer that container already runs on the +pod's loopback, so the sidecar must see the same env as the proxy. Under ``IAM_TOKEN_DB_AUTH`` or +``AZURE_POSTGRESQL_AUTH`` that PgBouncer only accepts the token the proxy container minted, so the +sidecar goes to Postgres directly and mints its own. Works from any image that has ``litellm`` +installed: + + python -m litellm.proxy.collector [--address unix:///path.sock] +""" + +import asyncio +import logging +import os +import signal +import sys +from collections.abc import Awaitable, Callable, Mapping, Sequence +from pathlib import Path +from typing import Final + +from litellm._logging import verbose_logger, verbose_proxy_logger, verbose_router_logger +from litellm.proxy.db.db_url_settings import DatabaseURLSettings +from litellm.proxy.db.pgbouncer import ( + PgBouncerError, + PgBouncerSettings, + export_pooled_database_url, + pooled_database_url, +) +from litellm.proxy.spend_tracking.spend_event_producer import ( + COLLECTOR_JOB_ROLE, + AddressError, + CollectorAddress, + CollectorSettings, + TcpAddress, + UnixAddress, + parse_collector_address, +) + +MAX_EVENT_BYTES: Final = 64 * 1024 * 1024 + + +class SpendEventConsumer: + """Accepts producer connections and runs ``handler`` on every line each one sends, in order.""" + + def __init__(self, handler: Callable[[bytes], Awaitable[None]]) -> None: + self._handler = handler + self._open_connections: set[asyncio.StreamWriter] = set() # mutable-ok: live producer connections + self._idle = asyncio.Event() + self._idle.set() + self._received = 0 + self._handled = 0 + self._failed = 0 + + @property + def received(self) -> int: + return self._received + + @property + def handled(self) -> int: + return self._handled + + @property + def failed(self) -> int: + return self._failed + + async def serve(self, address: CollectorAddress) -> asyncio.Server: + match address: + case UnixAddress(path=path): + socket_path: Final = Path(path) + socket_path.parent.mkdir(parents=True, exist_ok=True) + socket_path.unlink(missing_ok=True) + return await asyncio.start_unix_server(self._on_connection, path=path, limit=MAX_EVENT_BYTES) + case TcpAddress(host=host, port=port): + return await asyncio.start_server(self._on_connection, host=host, port=port, limit=MAX_EVENT_BYTES) + + async def _on_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self._open_connections.add(writer) + self._idle.clear() + try: + while line := await reader.readline(): + if not line.endswith(b"\n"): + verbose_proxy_logger.error("collector: discarding truncated spend event (%d bytes)", len(line)) + break + self._received += 1 + await self._handle(line) + except (ConnectionError, asyncio.IncompleteReadError, asyncio.LimitOverrunError) as error: + verbose_proxy_logger.warning("collector: producer connection ended abnormally: %s", error) + finally: + writer.close() + self._open_connections.discard(writer) + if not self._open_connections: + self._idle.set() + + async def _handle(self, line: bytes) -> None: + try: + await self._handler(line) + self._handled += 1 + except Exception: # noqa: BLE001 # the cost pipeline raises anything; one bad event must not stop the sidecar + self._failed += 1 + verbose_proxy_logger.exception("collector: spend event failed") + + async def drain(self, timeout: float) -> int: + """Half-close every producer connection, then keep reading until each producer hangs up or ``timeout``. + + Returns how many producer connections were still open when the timeout hit. + """ + for writer in tuple(self._open_connections): + if writer.is_closing() or not writer.can_write_eof(): + continue + try: + writer.write_eof() + except (OSError, RuntimeError) as error: + verbose_proxy_logger.debug("collector: producer already gone before half-close: %s", error) + try: + await asyncio.wait_for(self._idle.wait(), timeout) + except TimeoutError: + pass + return len(self._open_connections) + + +def _install_stop_signals(loop: asyncio.AbstractEventLoop, stop: asyncio.Event) -> None: + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop.set) + + +async def run_collector(address: CollectorAddress, drain_timeout: float) -> None: + from fastapi import FastAPI + + from litellm.proxy.hooks.proxy_track_cost_callback import run_spend_event + from litellm.proxy.proxy_server import proxy_startup_event + + stop: Final = asyncio.Event() + _install_stop_signals(asyncio.get_running_loop(), stop) + consumer: Final = SpendEventConsumer(handler=run_spend_event) + async with proxy_startup_event(FastAPI()): + server: Final = await consumer.serve(address) + verbose_proxy_logger.info("collector: listening on %s", address) + await stop.wait() + server.close() + still_open: Final = await consumer.drain(drain_timeout) + verbose_proxy_logger.info( + "collector: stopping. received=%d handled=%d failed=%d connections_cut=%d", + consumer.received, + consumer.handled, + consumer.failed, + still_open, + ) + + +def address_argument(argv: Sequence[str], default: str) -> str | AddressError: + match tuple(argv): + case (): + return default + case ("--address", value): + return value + case _: + return AddressError(f"usage: python -m litellm.proxy.collector [--address ADDRESS], got {tuple(argv)}") + + +def apply_log_level(litellm_log: str | None) -> None: + """Mirror the proxy's ``LITELLM_LOG`` handling: the sidecar has no CLI flags to turn logging on.""" + level: Final = logging.getLevelNamesMapping().get((litellm_log or "").upper()) + if level is None: + return + for logger in (verbose_logger, verbose_router_logger, verbose_proxy_logger): + logger.setLevel(level) + + +def pod_pgbouncer_database_url( + pgbouncer: PgBouncerSettings, environ: Mapping[str, str], *, token_auth: bool +) -> str | PgBouncerError | None: + """The proxy container's PgBouncer URL for ``environ["DATABASE_URL"]``, or None to connect to Postgres directly. + + Direct is the answer when PgBouncer is off, and also under token auth: that PgBouncer's auth file + only holds the token its own container minted, which this container cannot present. + """ + if not pgbouncer.enabled or token_auth: + return None + upstream_url: Final = environ.get("DATABASE_URL") + if upstream_url is None: + return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled") + return pooled_database_url(upstream_url, pgbouncer) + + +def main(argv: Sequence[str]) -> None: + os.environ.setdefault("LITELLM_JOB_ROLE", COLLECTOR_JOB_ROLE) + apply_log_level(os.environ.get("LITELLM_LOG")) + database: Final = DatabaseURLSettings.from_env() + database.apply_to_env() + pooled: Final = pod_pgbouncer_database_url( + PgBouncerSettings(), + os.environ, + token_auth=database.iam_token_db_auth or database.azure_postgresql_auth, + ) + if isinstance(pooled, PgBouncerError): + sys.exit(f"LiteLLM collector: cannot use the pod's pgbouncer: {pooled.reason}") + if pooled is not None: + export_pooled_database_url(pooled) + settings: Final = CollectorSettings() + raw_address: Final = address_argument(argv, default=settings.address) + address: Final = raw_address if isinstance(raw_address, AddressError) else parse_collector_address(raw_address) + if isinstance(address, AddressError): + sys.exit(f"LiteLLM collector: {address.reason}") + asyncio.run(run_collector(address, drain_timeout=settings.drain_timeout_seconds)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py index eac80f7a712..c9fbabd3585 100644 --- a/litellm/proxy/db/pgbouncer.py +++ b/litellm/proxy/db/pgbouncer.py @@ -280,6 +280,22 @@ def plan_pgbouncer( ) +def pooled_database_url(upstream_url: str, settings: PgBouncerSettings) -> str | PgBouncerError: + """The loopback URL of a PgBouncer another container in the pod already runs for ``upstream_url``. + + Only the container that started PgBouncer knows the pool user's password, so + this logs in as the upstream user, whom the auth file lists as well. + """ + plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir=Path("/nonexistent"), run_as_user=None) + if isinstance(plan, PgBouncerError): + return plan + password: Final = urllib.parse.urlsplit(upstream_url).password or "" + credentials: Final = f"{urllib.parse.quote(plan.upstream_user, safe='')}:{password}" + return urllib.parse.urlunsplit( + urllib.parse.urlsplit(plan.pooled_url)._replace(netloc=f"{credentials}@{PGBOUNCER_LISTEN_ADDR}:{settings.port}") + ) + + def _write_private(path: Path, content: str, run_as_user: str | None) -> None: with open(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w", encoding="utf-8") as handle: handle.write(content) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index c4fba8ecf9e..00406ad436e 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -27,6 +27,16 @@ from litellm.proxy.db.db_spend_update_writer import ( get_llm_router, ) from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.spend_tracking.spend_event import ( + ObjectMapping, + SpendEventBuildError, + SpendEventDecodeError, + build_spend_event, + decode_spend_event, + is_offloadable_success, + spend_event_callback_args, +) +from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer from litellm.proxy.spend_tracking.spend_log_error_logger import ( should_suppress_spend_log_tracebacks, spend_log_error, @@ -34,6 +44,7 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import ( from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_error_information_for_spend_logs, get_request_model_access_groups, + should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import ProxyUpdateSpend from litellm.types.utils import ( @@ -71,8 +82,43 @@ _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset( class _ProxyDBLogger(CustomLogger): - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time) + def __init__( + self, + spend_event_producer: SpendEventProducer | None = None, + *, + turn_off_message_logging: bool = False, + message_logging: bool = True, + ) -> None: + super().__init__(turn_off_message_logging=turn_off_message_logging, message_logging=message_logging) + self.spend_event_producer = spend_event_producer + + async def async_log_success_event( + self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + if self.spend_event_producer is None or not is_offloadable_success(response_obj): + await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time) + return + event: Final = build_spend_event( + kwargs, + response_obj, + start_time, + end_time, + store_bodies=should_store_prompts_and_responses_in_spend_logs(), + ) + if isinstance(event, SpendEventBuildError): + verbose_proxy_logger.warning("collector: tracking cost in-process, event not buildable: %s", event.reason) + await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time) + return + await self.spend_event_producer.publish(event) + + async def run_spend_event(self, line: bytes) -> None: + """Run the unchanged cost pipeline on a serialized spend event (sidecar consumer and in-process fallback).""" + event: Final = decode_spend_event(line) + if isinstance(event, SpendEventDecodeError): + verbose_proxy_logger.error("collector: discarding undecodable spend event: %s", event.reason) + return + args: Final = spend_event_callback_args(event) + await self._PROXY_track_cost_callback(args.kwargs, args.response_obj, args.start_time, args.end_time) async def async_post_call_failure_hook( self, @@ -503,6 +549,10 @@ def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None: bucket[key] = value +async def run_spend_event(line: bytes) -> None: + await _ProxyDBLogger().run_spend_event(line) + + def _is_unbilled_interaction_response(completion_response: object) -> bool: from litellm.interactions.background_cost_polling import missing_usage_is_expected from litellm.types.interactions import InteractionsAPIResponse diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7b64f65c79b..0f24acb8bb4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -469,7 +469,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( from litellm.proxy.hooks.prompt_injection_detection import ( _OPTIONAL_PromptInjectionDetection, ) -from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger +from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger, run_spend_event from litellm.proxy.image_endpoints.endpoints import router as image_router from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, @@ -592,6 +592,11 @@ from litellm.proxy.plugin_routes import ( from litellm.proxy.plugin_routes import ( router as plugin_router, ) +from litellm.proxy.spend_tracking.spend_event_producer import ( + CollectorSettings, + SpendEventProducer, + build_spend_event_producer, +) from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail try: @@ -943,6 +948,17 @@ def cleanup_router_config_variables(): heuristic_v1_tuning_baselines = None +async def flush_spend_counters_on_shutdown() -> None: + if prisma_client is None: + return + try: + await proxy_logging_obj.db_spend_update_writer.db_update_spend_transaction_handler( + prisma_client=prisma_client, n_retry_times=3, proxy_logging_obj=proxy_logging_obj + ) + except Exception as e: # noqa: BLE001 # shutdown must continue even if the commit fails + verbose_proxy_logger.exception("Error flushing spend counters on shutdown: %s", e) + + async def _flush_spend_logs_queue_on_shutdown() -> None: if prisma_client is None: return @@ -1378,6 +1394,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: except Exception as e: verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e) + await _drain_spend_event_producer_on_shutdown() + + await flush_spend_counters_on_shutdown() + await _flush_spend_logs_queue_on_shutdown() await proxy_config.stop_config_sync_subscriber() @@ -2457,16 +2477,27 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False): ) +spend_event_producer: SpendEventProducer | None = None + + def cost_tracking(): - global prisma_client + global prisma_client, spend_event_producer if prisma_client is not None: from litellm.integrations.shadow_eval_logger import ShadowEvalLogger - litellm.logging_callback_manager.add_litellm_callback(_ProxyDBLogger()) - litellm.logging_callback_manager.add_litellm_async_success_callback(_ProxyDBLogger()) + spend_event_producer = build_spend_event_producer(CollectorSettings(), fallback=run_spend_event) + litellm.logging_callback_manager.add_litellm_callback(_ProxyDBLogger(spend_event_producer)) + litellm.logging_callback_manager.add_litellm_async_success_callback(_ProxyDBLogger(spend_event_producer)) litellm.logging_callback_manager.add_litellm_callback(ShadowEvalLogger()) +async def _drain_spend_event_producer_on_shutdown() -> None: + if spend_event_producer is None: + return + await spend_event_producer.close(drain_timeout=CollectorSettings().drain_timeout_seconds) + verbose_proxy_logger.info("collector: producer drained on shutdown. stats=%s", spend_event_producer.stats()) + + # Bounds authoritative DB re-reads when enforcing a budget against a # stale-low spend counter: at most one DB read per counter per window. SPEND_DB_FLOOR_CACHE_TTL_SECONDS: Final = 5 diff --git a/litellm/proxy/spend_tracking/spend_event.py b/litellm/proxy/spend_tracking/spend_event.py new file mode 100644 index 00000000000..53f26346f85 --- /dev/null +++ b/litellm/proxy/spend_tracking/spend_event.py @@ -0,0 +1,418 @@ +"""Compact, typed success event handed from an inference worker to the collector. + +``build_spend_event`` runs on the inference worker right after ``Logging.async_success_handler`` +has built the ``standard_logging_object`` (so the cost is already known). It validates the success +callback's ``kwargs`` into the projection ``_PROXY_track_cost_callback`` and +``DBSpendUpdateWriter.update_database`` actually read: identities and metadata, timings, usage, the +standard logging payload without its prompt/response bodies, and the tool names. The request +messages, the raw ``proxy_server_request`` body and the full response travel only when spend logs +are configured to store prompts and responses. The cache key is the preset key the caching layer +already computed, never a fresh hash over the request body. + +``spend_event_callback_args`` rebuilds the ``(kwargs, response_obj, start_time, end_time)`` tuple +the existing cost pipeline consumes, so the sidecar runs the unchanged pipeline against the event. +""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from typing_extensions import NotRequired, ReadOnly, TypedDict + +import litellm +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.db.spend_log_tool_index import response_tool_call_names +from litellm.types.interactions import InteractionsAPIResponse +from litellm.types.utils import LiteLLMBatch, Usage + +SPEND_EVENT_VERSION: Final = 1 +CACHE_OFF_KEY: Final = "Cache OFF" + +ObjectMapping: TypeAlias = Mapping[str, object] + +_UNSERIALIZABLE_METADATA_KEYS: Final = frozenset({"user_api_key_auth", "litellm_parent_otel_span"}) +_STANDARD_LOGGING_BODY_KEYS: Final = frozenset({"messages", "response"}) +_STANDARD_LOGGING_DROPPED_KEYS: Final = frozenset({"model_parameters"}) +_NOT_OFFLOADED_RESPONSE_TYPES: Final = (LiteLLMBatch, InteractionsAPIResponse) + + +class _LitellmParams(TypedDict, total=False): + api_base: ReadOnly[str | None] + custom_llm_provider: ReadOnly[str | None] + litellm_call_id: ReadOnly[str | None] + user_api_key_end_user_id: ReadOnly[str | None] + metadata: ReadOnly[ObjectMapping | None] + litellm_metadata: ReadOnly[ObjectMapping | None] + proxy_server_request: ReadOnly[ObjectMapping | None] + preset_cache_key: ReadOnly[str | None] + + +class _DynamicParams(TypedDict, total=False): + turn_off_message_logging: ReadOnly[bool | None] + + +class _RequestBody(TypedDict, total=False): + tools: ReadOnly[Sequence[ObjectMapping] | None] + + +class _PassthroughPayload(TypedDict, total=False): + request_body: ReadOnly[_RequestBody | None] + + +class _ToolCallFunction(TypedDict): + name: ReadOnly[str] + arguments: ReadOnly[str] + + +class _ToolCall(TypedDict): + id: ReadOnly[str | None] + type: ReadOnly[Literal["function"]] + function: ReadOnly[_ToolCallFunction] + + +class _ToolCallMessage(TypedDict): + role: ReadOnly[Literal["assistant"]] + content: ReadOnly[None] + tool_calls: ReadOnly[Sequence[_ToolCall]] + + +class _ToolCallChoice(TypedDict): + index: ReadOnly[int] + finish_reason: ReadOnly[Literal["tool_calls"]] + message: ReadOnly[_ToolCallMessage] + + +class CompactResponse(TypedDict, total=False): + """What the spend pipeline reads off a response: its id, usage and which tools it called.""" + + id: ReadOnly[object] + model: ReadOnly[object] + usage: ReadOnly[object] + usage_info: ReadOnly[object] + status: ReadOnly[object] + background: ReadOnly[object] + choices: ReadOnly[Sequence[_ToolCallChoice]] + + +class _SuccessKwargs(TypedDict, total=False): + """The success callback's ``kwargs`` (``Logging.model_call_details``), validated and projected.""" + + litellm_call_id: ReadOnly[str | None] + call_type: ReadOnly[str | None] + model: ReadOnly[str | None] + custom_llm_provider: ReadOnly[str | None] + stream: ReadOnly[bool | None] + complete_streaming_response: ReadOnly[object] + cache_hit: ReadOnly[bool | None] + response_cost: ReadOnly[float | None] + completion_start_time: ReadOnly[datetime | None] + agent_id: ReadOnly[str | None] + litellm_trace_id: ReadOnly[str | None] + litellm_params: ReadOnly[_LitellmParams] + standard_logging_object: ReadOnly[ObjectMapping | None] + standard_callback_dynamic_params: ReadOnly[_DynamicParams | None] + combined_usage_object: ReadOnly[Usage | None] + realtime_tools: ReadOnly[Sequence[object] | None] + realtime_tool_calls: ReadOnly[Sequence[object] | None] + tools: ReadOnly[Sequence[ObjectMapping] | None] + passthrough_logging_payload: ReadOnly[_PassthroughPayload | None] + + +class _FunctionToolFunction(TypedDict): + name: ReadOnly[str] + + +class _FunctionTool(TypedDict): + type: ReadOnly[Literal["function"]] + function: ReadOnly[_FunctionToolFunction] + + +class SpendCallbackKwargs(TypedDict): + """The ``kwargs`` handed to ``_PROXY_track_cost_callback`` on the sidecar.""" + + litellm_call_id: ReadOnly[str | None] + call_type: ReadOnly[str | None] + model: ReadOnly[str | None] + custom_llm_provider: ReadOnly[str | None] + stream: ReadOnly[bool | None] + cache_hit: ReadOnly[bool | None] + response_cost: ReadOnly[float | None] + completion_start_time: ReadOnly[datetime | None] + agent_id: ReadOnly[str | None] + litellm_trace_id: ReadOnly[str | None] + litellm_params: ReadOnly[_LitellmParams] + standard_logging_object: ReadOnly[ObjectMapping | None] + standard_callback_dynamic_params: ReadOnly[_DynamicParams | None] + combined_usage_object: ReadOnly[Usage | None] + realtime_tools: ReadOnly[Sequence[object] | None] + realtime_tool_calls: ReadOnly[Sequence[object] | None] + tools: ReadOnly[Sequence[_FunctionTool] | None] + complete_streaming_response: NotRequired[ReadOnly[CompactResponse | None]] + + +_NO_LITELLM_PARAMS: Final[_LitellmParams] = {} +_SUCCESS_KWARGS: Final = TypeAdapter(_SuccessKwargs) +_OBJECT_MAPPING: Final = TypeAdapter(ObjectMapping) +_COMPACT_RESPONSE: Final = TypeAdapter(CompactResponse) + + +class SpendEvent(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + version: Literal[1] + litellm_call_id: str | None + call_type: str | None + model: str | None + custom_llm_provider: str | None + stream: bool | None + complete_streaming_response: bool + cache_hit: bool | None + response_cost: float | None + start_time: datetime + end_time: datetime + completion_start_time: datetime | None + agent_id: str | None + litellm_trace_id: str | None + litellm_params: _LitellmParams + standard_logging_object: ObjectMapping | None + standard_callback_dynamic_params: _DynamicParams | None + response: CompactResponse | None + combined_usage: ObjectMapping | None + realtime_tools: Sequence[object] | None + realtime_tool_calls: Sequence[object] | None + request_tool_names: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class SpendEventCallbackArgs: + kwargs: SpendCallbackKwargs + response_obj: CompactResponse | None + start_time: datetime + end_time: datetime + + +@dataclass(frozen=True, slots=True) +class SpendEventBuildError: + reason: str + + +@dataclass(frozen=True, slots=True) +class SpendEventDecodeError: + reason: str + + +def is_offloadable_success(response_obj: object) -> bool: + """Batch retrieves and interaction polls branch on the concrete response class, so they stay in-process.""" + return not isinstance(response_obj, _NOT_OFFLOADED_RESPONSE_TYPES) + + +def _json_fallback(value: object) -> str: + return str(value) + + +def _mapping_or_none(value: object) -> ObjectMapping | None: + try: + return _OBJECT_MAPPING.validate_python(value) + except ValidationError: + return None + + +def _drop_keys(mapping: ObjectMapping, keys: frozenset[str]) -> ObjectMapping: + return MappingProxyType({key: value for key, value in mapping.items() if key not in keys}) + + +def _budget_reservation(metadata: ObjectMapping) -> ObjectMapping | None: + """The admission-time reservation, wherever the request setup left it, so the sidecar can reconcile it.""" + direct: Final = _mapping_or_none(metadata.get("user_api_key_budget_reservation")) + if direct is not None: + return direct + auth: Final = metadata.get("user_api_key_auth") + if isinstance(auth, UserAPIKeyAuth): + return auth.budget_reservation + auth_mapping: Final = _mapping_or_none(auth) + return _mapping_or_none(auth_mapping.get("budget_reservation")) if auth_mapping is not None else None + + +def _metadata_for_event( + metadata: ObjectMapping | None, budget_reservation: ObjectMapping | None +) -> ObjectMapping | None: + if metadata is None: + return None + kept: Final = _drop_keys(metadata, _UNSERIALIZABLE_METADATA_KEYS) + if budget_reservation is None: + return kept + return MappingProxyType({**kept, "user_api_key_budget_reservation": budget_reservation}) + + +def _litellm_params_for_event( + litellm_params: _LitellmParams, cache_key: str | None, store_bodies: bool +) -> _LitellmParams: + metadata: Final = litellm_params.get("metadata") + litellm_metadata: Final = litellm_params.get("litellm_metadata") + budget_reservation: Final = next( + ( + reservation + for source in (litellm_metadata, metadata) + if source is not None and (reservation := _budget_reservation(source)) is not None + ), + None, + ) + projected: Final[_LitellmParams] = { + "api_base": litellm_params.get("api_base"), + "custom_llm_provider": litellm_params.get("custom_llm_provider"), + "litellm_call_id": litellm_params.get("litellm_call_id"), + "user_api_key_end_user_id": litellm_params.get("user_api_key_end_user_id"), + "metadata": _metadata_for_event(metadata, budget_reservation), + "litellm_metadata": _metadata_for_event(litellm_metadata, budget_reservation), + "proxy_server_request": litellm_params.get("proxy_server_request") if store_bodies else None, + "preset_cache_key": cache_key, + } + return projected + + +def _standard_logging_for_event(sl_object: ObjectMapping | None, store_bodies: bool) -> ObjectMapping | None: + if sl_object is None: + return None + dropped: Final = ( + _STANDARD_LOGGING_DROPPED_KEYS if store_bodies else _STANDARD_LOGGING_DROPPED_KEYS | _STANDARD_LOGGING_BODY_KEYS + ) + return _drop_keys(sl_object, dropped) + + +def _tool_call(name: str) -> _ToolCall: + tool_call: Final[_ToolCall] = {"id": None, "type": "function", "function": {"name": name, "arguments": "{}"}} + return tool_call + + +def _tool_call_choice(names: Sequence[str]) -> _ToolCallChoice: + choice: Final[_ToolCallChoice] = { + "index": 0, + "finish_reason": "tool_calls", + "message": {"role": "assistant", "content": None, "tool_calls": tuple(_tool_call(name) for name in names)}, + } + return choice + + +def _compact_response(response_obj: object) -> CompactResponse | None: + """Usage, identity and tool calls of the response, in chat-completions shape, without the content.""" + dumped: Final = response_obj.model_dump() if isinstance(response_obj, BaseModel) else _mapping_or_none(response_obj) + if dumped is None: + return None + scalars: Final = _COMPACT_RESPONSE.validate_python(_drop_keys(dumped, frozenset({"choices"}))) + tool_call_names: Final = response_tool_call_names(response_obj) + if not tool_call_names: + return scalars + with_tool_calls: Final[CompactResponse] = {**scalars, "choices": (_tool_call_choice(tool_call_names),)} + return with_tool_calls + + +def _tool_name(tool: ObjectMapping) -> str | None: + """Chat tools nest the name under ``function``; Anthropic and Responses API tools keep it at the top.""" + function: Final = _mapping_or_none(tool.get("function")) + name: Final = function.get("name") if function is not None else tool.get("name") + return name.strip() if isinstance(name, str) and name.strip() else None + + +def _request_tool_names(kwargs: _SuccessKwargs) -> tuple[str, ...]: + passthrough: Final = kwargs.get("passthrough_logging_payload") + request_body: Final = passthrough.get("request_body") if passthrough is not None else None + passthrough_tools: Final = request_body.get("tools") if request_body is not None else None + return tuple( + name + for source in (kwargs.get("tools"), passthrough_tools) + if source is not None + for tool in source + if (name := _tool_name(tool)) is not None + ) + + +def preset_spend_log_cache_key(litellm_params: _LitellmParams) -> str | None: + """The key the caching layer already stored in ``litellm_params``, or ``Cache OFF``; never hashes the body.""" + if litellm.cache is None: + return CACHE_OFF_KEY + return litellm_params.get("preset_cache_key") + + +def _function_tool(name: str) -> _FunctionTool: + tool: Final[_FunctionTool] = {"type": "function", "function": {"name": name}} + return tool + + +def build_spend_event( + raw_kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime, store_bodies: bool +) -> bytes | SpendEventBuildError: + """Validate the success callback's kwargs and serialize the event once, as a single JSON line.""" + try: + kwargs: Final = _SUCCESS_KWARGS.validate_python(raw_kwargs) + except ValidationError as error: + return SpendEventBuildError(reason=str(error)) + litellm_params: Final = kwargs.get("litellm_params", _NO_LITELLM_PARAMS) + sl_object: Final = kwargs.get("standard_logging_object") + cache_key: Final = preset_spend_log_cache_key(litellm_params) + response_cost: Final = sl_object.get("response_cost") if sl_object is not None else kwargs.get("response_cost") + combined_usage: Final = kwargs.get("combined_usage_object") + event: Final = SpendEvent( + version=SPEND_EVENT_VERSION, + litellm_call_id=kwargs.get("litellm_call_id"), + call_type=kwargs.get("call_type"), + model=kwargs.get("model"), + custom_llm_provider=kwargs.get("custom_llm_provider"), + stream=kwargs.get("stream"), + complete_streaming_response="complete_streaming_response" in kwargs, + cache_hit=kwargs.get("cache_hit"), + response_cost=response_cost if isinstance(response_cost, (int, float)) else None, + start_time=start_time, + end_time=end_time, + completion_start_time=kwargs.get("completion_start_time"), + agent_id=kwargs.get("agent_id"), + litellm_trace_id=kwargs.get("litellm_trace_id"), + litellm_params=_litellm_params_for_event(litellm_params, cache_key, store_bodies), + standard_logging_object=_standard_logging_for_event(sl_object, store_bodies), + standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params"), + response=_compact_response(response_obj), + combined_usage=combined_usage.model_dump() if combined_usage is not None else None, + realtime_tools=kwargs.get("realtime_tools"), + realtime_tool_calls=kwargs.get("realtime_tool_calls"), + request_tool_names=_request_tool_names(kwargs), + ) + return event.model_dump_json(fallback=_json_fallback).encode() + b"\n" + + +def decode_spend_event(line: bytes) -> SpendEvent | SpendEventDecodeError: + try: + return SpendEvent.model_validate_json(line) + except ValidationError as error: + return SpendEventDecodeError(reason=str(error)) + + +def spend_event_callback_args(event: SpendEvent) -> SpendEventCallbackArgs: + """The ``(kwargs, response_obj, start_time, end_time)`` the in-process cost callback receives.""" + tools: Final = tuple(_function_tool(name) for name in event.request_tool_names) + kwargs: Final[SpendCallbackKwargs] = { + "litellm_call_id": event.litellm_call_id, + "call_type": event.call_type, + "model": event.model, + "custom_llm_provider": event.custom_llm_provider, + "stream": event.stream, + "cache_hit": event.cache_hit, + "response_cost": event.response_cost, + "completion_start_time": event.completion_start_time, + "agent_id": event.agent_id, + "litellm_trace_id": event.litellm_trace_id, + "litellm_params": event.litellm_params, + "standard_logging_object": event.standard_logging_object, + "standard_callback_dynamic_params": event.standard_callback_dynamic_params, + "combined_usage_object": Usage.model_validate(event.combined_usage) + if event.combined_usage is not None + else None, + "realtime_tools": event.realtime_tools, + "realtime_tool_calls": event.realtime_tool_calls, + "tools": tools or None, + } + if not event.complete_streaming_response: + return SpendEventCallbackArgs(kwargs, event.response, event.start_time, event.end_time) + streaming_kwargs: Final[SpendCallbackKwargs] = {**kwargs, "complete_streaming_response": event.response} + return SpendEventCallbackArgs(streaming_kwargs, event.response, event.start_time, event.end_time) diff --git a/litellm/proxy/spend_tracking/spend_event_producer.py b/litellm/proxy/spend_tracking/spend_event_producer.py new file mode 100644 index 00000000000..20c7f177af0 --- /dev/null +++ b/litellm/proxy/spend_tracking/spend_event_producer.py @@ -0,0 +1,338 @@ +"""Fire-and-forget push of serialized spend events from an inference worker to the pod-local sidecar. + +``LITELLM_COLLECTOR_ENABLED=true`` turns the push on in the gateway; the sidecar process sets +``LITELLM_JOB_ROLE=collector`` and always runs the pipeline in-process. Events queue in a bounded +in-memory buffer that a single writer task flushes over a unix socket or loopback TCP connection. +When the sidecar is unreachable, the buffer is full, or the connection breaks mid-write, each affected +event follows ``LITELLM_COLLECTOR_ON_UNAVAILABLE``: ``fallback`` runs the existing cost pipeline in +the worker, ``drop`` counts it and moves on. Transitions are logged with the counters, so a sidecar +outage is visible without scraping anything. + +Delivery is at-most-once: a sidecar crash loses the events the kernel already took from its socket. +A sidecar that stops gracefully half-closes each connection first (EOF towards the producer) and +keeps reading until the producer hangs up, so the producer switches to the unavailable policy without +losing the events in flight. A write that fails part-way follows the unavailable policy without double +counting: ``drain()`` only fails while part of the line is still buffered in this process, so the +sidecar can at most have read a truncated line, which it discards. When the gateway itself stops with +the writer stuck mid-send, only an event whose bytes are still in the producer's write buffer follows +the unavailable policy; the connection is aborted first so the sidecar discards the truncated line +instead of also counting it. Events from one uvicorn worker are handled in the order it produced them; +events from different workers interleave, exactly like the in-process callbacks do today. +""" + +import asyncio +import ipaddress +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Final, Literal, TypeAlias +from urllib.parse import urlsplit + +from pydantic import AliasChoices, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +from litellm._logging import verbose_proxy_logger + +COLLECTOR_ENV_PREFIX: Final = "LITELLM_COLLECTOR_" +COLLECTOR_JOB_ROLE: Final = "collector" +DEFAULT_COLLECTOR_ADDRESS: Final = "unix:///var/run/litellm/collector.sock" +RECONNECT_BACKOFF_SECONDS: Final = 1.0 +DROP_LOG_EVERY: Final = 1000 + +UnavailablePolicy: TypeAlias = Literal["fallback", "drop"] +PublishOutcome: TypeAlias = Literal["queued", "fallback", "dropped"] + + +class CollectorSettings(BaseSettings): + """``LITELLM_COLLECTOR_*`` env vars, shared by the gateway producer and the sidecar consumer.""" + + model_config = SettingsConfigDict( + env_prefix=COLLECTOR_ENV_PREFIX, case_sensitive=False, extra="ignore", frozen=True, populate_by_name=True + ) + + enabled: bool = False + address: str = DEFAULT_COLLECTOR_ADDRESS + buffer_size: int = Field(default=1000, ge=1) + on_unavailable: UnavailablePolicy = "fallback" + drain_timeout_seconds: float = Field(default=10.0, gt=0) + connect_timeout_seconds: float = Field(default=1.0, gt=0) + job_role: str | None = Field(default=None, validation_alias=AliasChoices("LITELLM_JOB_ROLE")) + + @property + def produces(self) -> bool: + return self.enabled and self.job_role != COLLECTOR_JOB_ROLE + + +@dataclass(frozen=True, slots=True) +class UnixAddress: + path: str + + +@dataclass(frozen=True, slots=True) +class TcpAddress: + host: str + port: int + + +@dataclass(frozen=True, slots=True) +class AddressError: + reason: str + + +CollectorAddress: TypeAlias = UnixAddress | TcpAddress + + +def _is_loopback(host: str) -> bool: + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return host == "localhost" + + +def parse_collector_address(address: str) -> CollectorAddress | AddressError: + """``unix:///path/to.sock`` or ``tcp://127.0.0.1:port``; the socket carries unauthenticated spend events.""" + parsed: Final = urlsplit(address) + if parsed.scheme == "unix" and parsed.path: + return UnixAddress(path=parsed.path) + if parsed.scheme == "tcp" and parsed.hostname and parsed.port is not None: + if not _is_loopback(parsed.hostname): + return AddressError(reason=f"tcp collector address must be a loopback host, got {address!r}") + return TcpAddress(host=parsed.hostname, port=parsed.port) + return AddressError(reason=f"expected unix:///path or tcp://127.0.0.1:port, got {address!r}") + + +async def open_collector_connection( + address: CollectorAddress, timeout: float +) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + match address: + case UnixAddress(path=path): + return await asyncio.wait_for(asyncio.open_unix_connection(path), timeout) + case TcpAddress(host=host, port=port): + return await asyncio.wait_for(asyncio.open_connection(host, port), timeout) + + +def build_spend_event_producer( + settings: CollectorSettings, fallback: Callable[[bytes], Awaitable[None]] +) -> "SpendEventProducer | None": + """The gateway producer for these settings, or ``None`` when the pipeline stays in-process.""" + if not settings.produces: + return None + address: Final = parse_collector_address(settings.address) + if isinstance(address, AddressError): + verbose_proxy_logger.error("collector: %s; running the spend pipeline in-process", address.reason) + return None + verbose_proxy_logger.info( + "collector: offloading spend tracking to %s (buffer=%d, on_unavailable=%s)", + settings.address, + settings.buffer_size, + settings.on_unavailable, + ) + return SpendEventProducer( + address=address, + on_unavailable=settings.on_unavailable, + buffer_size=settings.buffer_size, + connect_timeout=settings.connect_timeout_seconds, + fallback=fallback, + ) + + +@dataclass(frozen=True, slots=True) +class _Connection: + reader: asyncio.StreamReader + writer: asyncio.StreamWriter + + @property + def alive(self) -> bool: + return not self.writer.is_closing() and not self.reader.at_eof() + + +@dataclass(frozen=True, slots=True) +class SpendEventProducerStats: + queued: int + sent: int + fallback: int + dropped: int + connected: bool + + +class SpendEventProducer: + """Bounded buffer plus one writer task per process; see the module docstring for the contract.""" + + def __init__( + self, + address: CollectorAddress, + on_unavailable: UnavailablePolicy, + buffer_size: int, + connect_timeout: float, + fallback: Callable[[bytes], Awaitable[None]], + clock: Callable[[], float] = time.monotonic, + open_connection: Callable[ + [CollectorAddress, float], Awaitable[tuple[asyncio.StreamReader, asyncio.StreamWriter]] + ] = open_collector_connection, + ) -> None: + self._address = address + self._on_unavailable = on_unavailable + self._buffer_size = buffer_size + self._connect_timeout = connect_timeout + self._fallback = fallback + self._clock = clock + self._open_connection = open_connection + self._queue: asyncio.Queue[bytes] | None = None + self._writer_task: asyncio.Task[None] | None = None + self._connection: _Connection | None = None + self._in_flight: bytes | None = None + self._closing = False + self._next_connect_at = 0.0 + self._queued = 0 + self._sent = 0 + self._fallback_count = 0 + self._dropped = 0 + + def stats(self) -> SpendEventProducerStats: + return SpendEventProducerStats( + queued=self._queued, + sent=self._sent, + fallback=self._fallback_count, + dropped=self._dropped, + connected=self._connection is not None, + ) + + async def publish(self, line: bytes) -> PublishOutcome: + """Hand one serialized event to the writer task, or apply the unavailable policy right away.""" + if self._closing or self._clock() < self._next_connect_at: + return await self._unavailable(line, "sidecar unreachable") + queue: Final = self._ensure_writer() + try: + queue.put_nowait(line) + except asyncio.QueueFull: + return await self._unavailable(line, "buffer full") + self._queued += 1 + return "queued" + + async def close(self, drain_timeout: float) -> None: + """Flush the buffer for up to ``drain_timeout`` seconds, then apply the unavailable policy to the rest.""" + self._closing = True + queue: Final = self._queue + task: Final = self._writer_task + if queue is None or task is None: + return + try: + await asyncio.wait_for(queue.join(), drain_timeout) + except asyncio.TimeoutError: + verbose_proxy_logger.warning( + "collector: %s events still buffered after %.1fs drain timeout", queue.qsize(), drain_timeout + ) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + unsent: Final = self._take_unsent() + await self._disconnect() + if unsent is not None: + await self._unavailable(unsent, "shutdown") + while not queue.empty(): + await self._unavailable(queue.get_nowait(), "shutdown") + + def _take_unsent(self) -> bytes | None: + """The in-flight event if any of its bytes never left this process, aborting the half-written connection.""" + in_flight: Final = self._in_flight + self._in_flight = None + connection: Final = self._connection + if in_flight is None: + return None + if connection is None: + return in_flight + if connection.writer.transport.get_write_buffer_size() == 0: + return None + connection.writer.transport.abort() + return in_flight + + def _ensure_writer(self) -> asyncio.Queue[bytes]: + if self._queue is None: + self._queue = asyncio.Queue(maxsize=self._buffer_size) + if self._writer_task is None or self._writer_task.done(): + self._writer_task = asyncio.get_running_loop().create_task(self._run_writer(self._queue)) + return self._queue + + async def _run_writer(self, queue: asyncio.Queue[bytes]) -> None: + while True: + line = await queue.get() + try: + await self._send(line) + finally: + queue.task_done() + + async def _send(self, line: bytes) -> None: + self._in_flight = line + connection: Final = await self._connect() + if connection is None: + self._in_flight = None + await self._unavailable(line, "sidecar unreachable") + return + try: + connection.writer.write(line) + await connection.writer.drain() + except (ConnectionError, OSError, RuntimeError) as error: # uvloop: RuntimeError on a closed transport + self._in_flight = None + await self._disconnect() + self._next_connect_at = self._clock() + RECONNECT_BACKOFF_SECONDS + await self._unavailable(line, f"write failed: {error}") + return + self._in_flight = None + self._sent += 1 + + async def _connect(self) -> _Connection | None: + if self._connection is not None and self._connection.alive: + return self._connection + await self._disconnect() + if self._clock() < self._next_connect_at: + return None + try: + reader, writer = await self._open_connection(self._address, self._connect_timeout) + except (ConnectionError, OSError, asyncio.TimeoutError) as error: + self._next_connect_at = self._clock() + RECONNECT_BACKOFF_SECONDS + verbose_proxy_logger.warning( + "collector: cannot reach %s (%s); applying %s policy for %.0fs. stats=%s", + self._address, + error, + self._on_unavailable, + RECONNECT_BACKOFF_SECONDS, + self.stats(), + ) + return None + self._connection = _Connection(reader=reader, writer=writer) + verbose_proxy_logger.info("collector: connected to %s. stats=%s", self._address, self.stats()) + return self._connection + + async def _disconnect(self) -> None: + connection: Final = self._connection + self._connection = None + if connection is None: + return + connection.writer.close() + try: + await connection.writer.wait_closed() + except (ConnectionError, OSError): + pass + + async def _unavailable(self, line: bytes, reason: str) -> PublishOutcome: + if self._on_unavailable == "fallback": + self._fallback_count += 1 + fallback: Final = asyncio.ensure_future(self._run_fallback(line, reason)) + try: + await asyncio.shield(fallback) + except asyncio.CancelledError: + await fallback + raise + return "fallback" + self._dropped += 1 + if self._dropped % DROP_LOG_EVERY == 1: + verbose_proxy_logger.warning("collector: dropping spend event (%s). stats=%s", reason, self.stats()) + return "dropped" + + async def _run_fallback(self, line: bytes, reason: str) -> None: + try: + await self._fallback(line) + except Exception: # noqa: BLE001 # one failing event must not kill the writer task + verbose_proxy_logger.exception("collector: in-process fallback failed (%s)", reason) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index dad9a2a8777..f0f38358cf0 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -554,10 +554,12 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens clean_metadata["additional_usage_values"] = additional_usage_values - if litellm.cache is not None: - cache_key = litellm.cache.get_cache_key(**kwargs) - else: + if litellm.cache is None: cache_key = "Cache OFF" + elif litellm_params.get("preset_cache_key") is not None: + cache_key = litellm_params["preset_cache_key"] + else: + cache_key = litellm.cache.get_cache_key(**kwargs) if cache_hit is True: import time @@ -864,7 +866,7 @@ def _get_messages_for_spend_logs_payload( standard_logging_payload: StandardLoggingPayload | None, metadata: dict | None = None, ) -> str: - if _should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(): if standard_logging_payload is not None: call_type: Final = standard_logging_payload.get("call_type", "") if call_type == "_arealtime": @@ -1114,7 +1116,7 @@ def _sanitize_guardrail_information_for_spend_logs( here to match OTEL's defensive read pattern; otherwise iteration would yield the dict's keys and crash the whole spend-log write. """ - if guardrail_information is None or _should_store_prompts_and_responses_in_spend_logs(): + if guardrail_information is None or should_store_prompts_and_responses_in_spend_logs(): return guardrail_information entries: Final = [guardrail_information] if isinstance(guardrail_information, dict) else guardrail_information return [_redact_prompt_fields_in_guardrail_entry(entry) for entry in entries if isinstance(entry, dict)] @@ -1186,7 +1188,7 @@ def _sanitize_error_information_for_spend_logs( sanitized = cast(dict, {**error_information}) - if not _should_store_prompts_and_responses_in_spend_logs(): + if not should_store_prompts_and_responses_in_spend_logs(): for field in ("error_message", "traceback"): value = sanitized.get(field) if isinstance(value, str): @@ -1263,11 +1265,11 @@ def _get_proxy_server_request_for_spend_logs_payload( kwargs: dict | None = None, ) -> str: """ - Only store if _should_store_prompts_and_responses_in_spend_logs() is True + Only store if should_store_prompts_and_responses_in_spend_logs() is True If turn_off_message_logging is enabled, redact messages in the request body. """ - if _should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(): _proxy_server_request: Final = cast(dict | None, litellm_params.get("proxy_server_request", EMPTY_MAPPING)) if _proxy_server_request is not None: _request_body = _proxy_server_request.get("body", EMPTY_MAPPING) or EMPTY_MAPPING @@ -1317,7 +1319,7 @@ def _get_vector_store_request_for_spend_logs_payload( """ If user does not want to store prompts and responses, then remove the content from the vector store request metadata """ - if _should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(): return vector_store_request_metadata # if user does not want to store prompts and responses, then remove the content from the vector store request metadata @@ -1341,7 +1343,7 @@ def _get_response_for_spend_logs_payload( ) -> str: if payload is None: return "{}" - if _should_store_prompts_and_responses_in_spend_logs(): + if should_store_prompts_and_responses_in_spend_logs(): response_obj: object = payload.get("response") if response_obj is None: return "{}" @@ -1389,7 +1391,7 @@ def _get_response_for_spend_logs_payload( return "{}" -def _should_store_prompts_and_responses_in_spend_logs() -> bool: +def should_store_prompts_and_responses_in_spend_logs() -> bool: from litellm.proxy.proxy_server import general_settings from litellm.secret_managers.main import get_secret_bool diff --git a/litellm/utils.py b/litellm/utils.py index 69ee730ff30..aced4c9b312 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1843,6 +1843,9 @@ def client(original_function): elif _caching_handler_response.embedding_all_elements_cache_hit is True: return _caching_handler_response.final_embedding_cached_response + if _llm_caching_handler.preset_cache_key is not None: + logging_obj.litellm_params["preset_cache_key"] = _llm_caching_handler.preset_cache_key + # CHECK MAX TOKENS if ( kwargs.get("max_tokens", None) is not None diff --git a/terraform/litellm/README.md b/terraform/litellm/README.md index d4b40741052..7cc6e4c08ba 100644 --- a/terraform/litellm/README.md +++ b/terraform/litellm/README.md @@ -183,6 +183,7 @@ only where the underlying cloud forces it. | Extra secret-backed env | `gateway_extra_secrets`, `backend_extra_secrets` (ARNs) | `gateway_extra_secrets`, `backend_extra_secrets` (resource IDs) | | Uvicorn `--workers` on gateway | `gateway_num_workers` | `gateway_num_workers` | | OpenTelemetry v2 (opt-in) | `otel_endpoint`, `otel_exporter`, `otel_environment_name`, `otel_capture_message_content`, `otel_headers_secret_arn` | `otel_endpoint`, `otel_exporter`, `otel_environment_name`, `otel_capture_message_content`, `otel_headers_secret` | +| Collector sidecar (opt-in) | `collector_enabled`, `collector_port`, `collector_cpu`, `collector_memory`, `collector_buffer_size`, `collector_on_unavailable`, `collector_drain_timeout_seconds` | same names; `collector_cpu` / `collector_memory` take Cloud Run strings | Each module stamps its own stack-identity tag (`litellm:stack` on AWS, `litellm-stack` on GCP — GCP label keys forbid colons) plus diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 5ca45944c54..67f1270d0c2 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -345,6 +345,39 @@ ten tasks handle 4,200,000,000 tokens in a minute, `tokens / 60` is `ceil(10 * 7000000 / 6000000) = 12`. Container Insights must be enabled on the cluster for `RunningTaskCount` to exist +### Collector sidecar + +`collector_enabled = true` adds a second container to the gateway task +that runs `python -m litellm.proxy.collector` from the gateway image, and sets +`LITELLM_COLLECTOR_ENABLED=true` on the gateway so its uvicorn workers +ship spend events (SpendLogs writes, key/team/user spend updates, budget +alerts) to the sidecar instead of running that pipeline in the request +path. This is the Terraform counterpart of helm's `gateway.collector`. +The default (`false`) leaves the task definition exactly as before. + +Fargate tasks share one network namespace, so the sidecar listens on +loopback TCP (`tcp://127.0.0.1:${collector_port}`, default 4010) instead +of the Unix socket helm uses; the proxy rejects any non-loopback address. +The sidecar gets the same database, Redis, master-key, license, proxy +config, and `gateway_extra_env` / `gateway_extra_secrets` values as the +gateway container, runs with `LITELLM_JOB_ROLE=collector`, and is +non-essential with an ECS restart policy, so a sidecar crash restarts it in +place while the gateway falls back to in-process spend tracking. + +```hcl +collector_enabled = true +# collector_cpu = 512 # carved out of gateway_cpu +# collector_memory = 2048 # MiB, carved out of gateway_memory +# collector_buffer_size = 1000 +# collector_on_unavailable = "fallback" # or "drop" +# collector_drain_timeout_seconds = 10 +``` + +Both sidecar reservations must leave room for the gateway container inside +`gateway_cpu` / `gateway_memory` (the plan fails otherwise). Service +autoscaling keeps tracking the whole task's CPU and memory, sidecar +included + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 7176921e8fa..2b235c2bad5 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -278,6 +278,62 @@ locals { "${local.proxy_config_fetch_cmd} && ${local.backend_launch_cmd}" ] } : {} + + collector_address = "tcp://127.0.0.1:${var.collector_port}" + collector_env = var.collector_enabled ? [ + { name = "LITELLM_COLLECTOR_ENABLED", value = "true" }, + { name = "LITELLM_COLLECTOR_ADDRESS", value = local.collector_address }, + { name = "LITELLM_COLLECTOR_BUFFER_SIZE", value = tostring(var.collector_buffer_size) }, + { name = "LITELLM_COLLECTOR_ON_UNAVAILABLE", value = var.collector_on_unavailable }, + { name = "LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS", value = tostring(var.collector_drain_timeout_seconds) }, + ] : [] + + gateway_environment = concat( + local.shared_env, + local.gateway_otel_env, + local.billing_metrics_env, + local.gateway_extra_env_list, + local.proxy_config_env, + local.metrics_env, + local.gateway_pool_env, + local.collector_env, + ) + + collector_launch_cmd = "exec python -m litellm.proxy.collector" + collector_command = [ + local.proxy_config_enabled ? "${local.proxy_config_fetch_cmd} && ${local.collector_launch_cmd}" : local.collector_launch_cmd + ] + + collector_container = var.collector_enabled ? [{ + name = "collector" + image = var.gateway_image + essential = false + cpu = var.collector_cpu + memory = var.collector_memory + + restartPolicy = { enabled = true } + + entryPoint = ["sh", "-c"] + command = local.collector_command + environment = concat( + local.shared_env, + local.gateway_extra_env_list, + local.proxy_config_env, + local.gateway_pool_env, + local.collector_env, + [{ name = "LITELLM_JOB_ROLE", value = "collector" }], + ) + secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) + + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.gateway.name + awslogs-region = var.region + awslogs-stream-prefix = "collector" + } + } + }] : [] } # ---------- Gateway ---------- @@ -309,6 +365,16 @@ resource "aws_ecs_task_definition" "gateway" { condition = !var.gateway_connection_pool_enabled || local.database_enabled error_message = "gateway_connection_pool_enabled needs a database: set create_database = true or pass database_url." } + + precondition { + condition = !var.collector_enabled || (var.collector_cpu < var.gateway_cpu && var.collector_memory < var.gateway_memory) + error_message = "collector_cpu and collector_memory are carved out of gateway_cpu / gateway_memory and must leave room for the gateway container." + } + + precondition { + condition = !var.collector_enabled || var.gateway_metrics_port == null || var.collector_port != var.gateway_metrics_port + error_message = "collector_port and gateway_metrics_port must differ: both sidecars bind loopback in the same task." + } } family = "${local.name}-gateway" @@ -327,17 +393,9 @@ resource "aws_ecs_task_definition" "gateway" { essential = true portMappings = [{ containerPort = 4000, protocol = "tcp" }] - environment = concat( - local.shared_env, - local.gateway_otel_env, - local.billing_metrics_env, - local.gateway_extra_env_list, - local.proxy_config_env, - local.metrics_env, - local.gateway_pool_env, - ) - secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) - mountPoints = local.metrics_mount_points + environment = local.gateway_environment + secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list) + mountPoints = local.metrics_mount_points # Container-level healthCheck intentionally omitted — the wolfi # runtime image doesn't ship curl/wget. The ALB target group polls @@ -354,7 +412,7 @@ resource "aws_ecs_task_definition" "gateway" { }, local.gateway_proxy_overrides, ) - ], local.gateway_metrics_container)) + ], local.gateway_metrics_container, local.collector_container)) dynamic "volume" { for_each = local.metrics_enabled ? [1] : [] diff --git a/terraform/litellm/aws/tests/collector.tftest.hcl b/terraform/litellm/aws/tests/collector.tftest.hcl new file mode 100644 index 00000000000..1465130232f --- /dev/null +++ b/terraform/litellm/aws/tests/collector.tftest.hcl @@ -0,0 +1,144 @@ +# Plan-only coverage for the opt-in collector sidecar in the gateway task. +# The rendered container_definitions JSON is unknown at plan time (it embeds +# Aurora/ElastiCache endpoints and secret ARNs), so the assertions target the +# locals it is built from. Run from terraform/litellm/aws with `terraform test`. + +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} +mock_provider "random" {} + +variables { + region = "us-east-1" + tenant = "acme" + env = "test" + allow_plaintext_alb = true + azs = ["us-east-1a", "us-east-1b"] +} + +run "disabled_by_default_leaves_the_task_untouched" { + command = plan + + assert { + condition = length(local.collector_container) == 0 + error_message = "The gateway task must stay single-container unless collector_enabled is set." + } + + assert { + condition = !anytrue([for e in local.gateway_environment : startswith(e.name, "LITELLM_COLLECTOR_")]) + error_message = "No LITELLM_COLLECTOR_* env may reach the gateway while the sidecar is disabled." + } +} + +run "enabled_adds_a_sidecar_that_shares_the_gateway_transport" { + command = plan + + variables { + collector_enabled = true + collector_port = 4321 + collector_buffer_size = 250 + collector_on_unavailable = "drop" + gateway_extra_env = { OPENAI_API_BASE = "https://example.invalid" } + gateway_extra_secrets = { OPENAI_API_KEY = "arn:aws:secretsmanager:us-east-1:111122223333:secret:openai-AbCdEf" } + } + + assert { + condition = length(local.collector_container) == 1 && local.collector_container[0].name == "collector" + error_message = "Enabling the sidecar must add exactly one collector container." + } + + assert { + condition = alltrue([ + for env in [local.gateway_environment, local.collector_container[0].environment] : ( + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_ENABLED"] == "true" && + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_ADDRESS"] == "tcp://127.0.0.1:4321" && + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_BUFFER_SIZE"] == "250" && + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_ON_UNAVAILABLE"] == "drop" && + { for e in env : e.name => e.value }["LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS"] == "10" + ) + ]) + error_message = "Gateway and sidecar must agree on the loopback address and the collector knobs." + } + + assert { + condition = ( + local.collector_container[0].image == var.gateway_image && + local.collector_container[0].entryPoint == ["sh", "-c"] && + local.collector_container[0].command == ["exec python -m litellm.proxy.collector"] && + local.collector_container[0].essential == false && + local.collector_container[0].restartPolicy.enabled == true && + { for e in local.collector_container[0].environment : e.name => e.value }["LITELLM_JOB_ROLE"] == "collector" + ) + error_message = "The sidecar must run litellm.proxy.collector from the gateway image as a restartable, non-essential collector." + } + + assert { + condition = ( + { for e in local.collector_container[0].environment : e.name => e.value }["OPENAI_API_BASE"] == "https://example.invalid" && + contains([for e in local.collector_container[0].environment : e.name], "DATABASE_HOST") && + contains([for e in local.collector_container[0].environment : e.name], "REDIS_HOST") && + contains([for s in local.collector_container[0].secrets : s.name], "LITELLM_MASTER_KEY") && + contains([for s in local.collector_container[0].secrets : s.name], "OPENAI_API_KEY") + ) + error_message = "The sidecar must receive the gateway's database, Redis, and shared secrets plus gateway_extra_env / gateway_extra_secrets." + } + + assert { + condition = !contains(keys(local.collector_container[0]), "portMappings") + error_message = "The sidecar must not expose a port to the task's load balancer." + } + + assert { + condition = local.collector_container[0].cpu == 512 && local.collector_container[0].memory == 2048 + error_message = "The sidecar defaults must mirror helm's collector resources (500m / 2Gi)." + } +} + +run "proxy_config_is_fetched_by_the_sidecar_too" { + command = plan + + variables { + collector_enabled = true + proxy_config = { model_list = [] } + } + + assert { + condition = ( + startswith(local.collector_container[0].command[0], local.proxy_config_fetch_cmd) && + endswith(local.collector_container[0].command[0], "exec python -m litellm.proxy.collector") && + contains([for e in local.collector_container[0].environment : e.name], "CONFIG_FILE_PATH") + ) + error_message = "The sidecar must pull the proxy config from S3 before starting, like the gateway does." + } +} + +run "sidecar_must_leave_room_for_the_gateway" { + command = plan + + variables { + collector_enabled = true + collector_cpu = 1024 + } + + expect_failures = [ + aws_ecs_task_definition.gateway, + ] +} + +run "sidecars_must_not_share_a_loopback_port" { + command = plan + + variables { + collector_enabled = true + collector_port = 4001 + gateway_metrics_port = 4001 + } + + expect_failures = [ + aws_ecs_task_definition.gateway, + ] +} diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index f57bb1ab7f6..580a0cc657a 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -808,3 +808,73 @@ variable "billing_metrics_ca_cert_pem" { default = "" sensitive = true } + +# ---------- Collector sidecar ---------- +# +# Opt-in offload of spend tracking from the gateway's uvicorn workers to a +# `python -m litellm.proxy.collector` sidecar in the same Fargate task (helm's +# `gateway.collector`). Fargate awsvpc tasks share one network namespace, +# so the sidecar listens on loopback TCP. Disabled (the default) adds nothing +# to the task definition. + +variable "collector_enabled" { + description = "Run the collector sidecar next to the gateway container and have the gateway ship spend events to it (sets LITELLM_COLLECTOR_ENABLED=true on both). Autoscaling still targets the whole task's CPU/memory, sidecar included." + type = bool + default = false +} + +variable "collector_port" { + description = "Loopback TCP port the sidecar listens on (LITELLM_COLLECTOR_ADDRESS=tcp://127.0.0.1:)." + type = number + default = 4010 + + validation { + condition = var.collector_port >= 1024 && var.collector_port <= 65535 && var.collector_port != 4000 + error_message = "collector_port must be in 1024-65535 and not 4000." + } +} + +variable "collector_cpu" { + description = "CPU units reserved for the sidecar container, carved out of gateway_cpu. Matches helm's collector.resources.requests.cpu (500m)." + type = number + default = 512 +} + +variable "collector_memory" { + description = "Hard memory limit (MiB) for the sidecar container, carved out of gateway_memory. Matches helm's collector.resources.limits.memory (2Gi)." + type = number + default = 2048 +} + +variable "collector_buffer_size" { + description = "Per-worker in-memory queue of spend events waiting to be shipped to the sidecar (LITELLM_COLLECTOR_BUFFER_SIZE)." + type = number + default = 1000 + + validation { + condition = var.collector_buffer_size >= 1 + error_message = "collector_buffer_size must be >= 1." + } +} + +variable "collector_on_unavailable" { + description = "What the gateway does with spend events when the sidecar is unreachable or the buffer is full (LITELLM_COLLECTOR_ON_UNAVAILABLE): `fallback` runs the pipeline in-process, `drop` discards them." + type = string + default = "fallback" + + validation { + condition = contains(["fallback", "drop"], var.collector_on_unavailable) + error_message = "collector_on_unavailable must be one of: fallback, drop." + } +} + +variable "collector_drain_timeout_seconds" { + description = "Seconds a gateway worker waits on shutdown for its buffered spend events to reach the sidecar (LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS)." + type = number + default = 10 + + validation { + condition = var.collector_drain_timeout_seconds > 0 + error_message = "collector_drain_timeout_seconds must be > 0." + } +} diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 9c71d3e15b7..23cfe9afc24 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -321,6 +321,43 @@ launcher reads these variables, starts the pooler once per instance before uvicorn forks the workers and hands them its loopback `DATABASE_URL`. It also honours `KEEPALIVE_TIMEOUT` from `gateway_extra_env` the way the image does +### Collector sidecar + +`collector_enabled = true` adds a `spend-collector` container to the gateway +Cloud Run service that runs `python -m litellm.proxy.collector` from the gateway +image, and sets `LITELLM_COLLECTOR_ENABLED=true` on the gateway so its +uvicorn workers ship spend events (SpendLogs writes, key/team/user spend +updates, budget alerts) to the sidecar instead of running that pipeline in +the request path. This is the Terraform counterpart of helm's +`gateway.collector`. The default (`false`) leaves the service exactly as +before. It is independent of the metrics sidecars above, whose GMP scraper +already owns the `collector` container name. + +Containers in one Cloud Run instance share localhost, so the sidecar listens +on loopback TCP (`tcp://127.0.0.1:${collector_port}`, default 4010) +instead of the Unix socket helm uses; the proxy rejects any non-loopback +address. The sidecar runs the same Redis CA + `DATABASE_URL` bootstrap as +the gateway container, gets the same database, Redis, master-key, license, +proxy config, and `gateway_extra_env` / `gateway_extra_secrets` values, and +runs with `LITELLM_JOB_ROLE=collector`. When it is unreachable the +gateway falls back to in-process spend tracking. + +```hcl +collector_enabled = true +# collector_cpu = "1000m" # added on top of gateway_cpu +# collector_memory = "2Gi" # added on top of gateway_memory +# collector_buffer_size = 1000 +# collector_on_unavailable = "fallback" # or "drop" +# collector_drain_timeout_seconds = 10 +``` + +Cloud Run allocates CPU per instance while requests are in flight, and the +sidecar shares that allocation. Spend events are shipped right after each +response, so this works with request-based billing, but keep +`gateway_min_instances >= 1` if spend must keep draining while an instance +is otherwise idle. Variable names match the AWS stack; only the resource +units differ (Cloud Run strings vs Fargate units) + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index 78d4ffa2152..d0b32a367d6 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -177,6 +177,34 @@ locals { [local.backend_launch_cmd], )) + collector_address = "tcp://127.0.0.1:${var.collector_port}" + collector_env_kv = var.collector_enabled ? [ + { name = "LITELLM_COLLECTOR_ENABLED", value = "true" }, + { name = "LITELLM_COLLECTOR_ADDRESS", value = local.collector_address }, + { name = "LITELLM_COLLECTOR_BUFFER_SIZE", value = tostring(var.collector_buffer_size) }, + { name = "LITELLM_COLLECTOR_ON_UNAVAILABLE", value = var.collector_on_unavailable }, + { name = "LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS", value = tostring(var.collector_drain_timeout_seconds) }, + ] : [] + + gateway_env_kv = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env, local.metrics_env_kv, local.gateway_pool_env, local.collector_env_kv) + gateway_env_secrets = concat(local.shared_env_secrets, local.otel_env_secrets, local.billing_metrics_env_secrets, local.gateway_extra_secret_kv) + + collector_env_kv_all = concat( + local.shared_env_kv, + local.gateway_extra_env_kv, + local.proxy_config_env, + local.gateway_pool_env, + local.collector_env_kv, + [{ name = "LITELLM_JOB_ROLE", value = "collector" }], + ) + collector_env_secrets = concat(local.shared_env_secrets, local.gateway_extra_secret_kv) + + collector_args = join(" && ", concat( + local.redis_ca_fragment, + local.database_url_fragment, + ["exec python -m litellm.proxy.collector"], + )) + # Env shipped to the migrations Job. The migrations image runs run.py # which assembles DATABASE_URL from these discrete vars itself, so we # only need writer-side DB env (no read replica, no proxy_config, no @@ -203,6 +231,13 @@ resource "google_cloud_run_v2_service" "gateway" { labels = local.labels deletion_protection = false + lifecycle { + precondition { + condition = !var.collector_enabled || var.gateway_metrics_port == null || var.collector_port != var.gateway_metrics_port + error_message = "collector_port and gateway_metrics_port must differ: both sidecars bind loopback in the same instance." + } + } + template { service_account = google_service_account.runtime.email max_instance_request_concurrency = var.gateway_max_instance_request_concurrency @@ -235,7 +270,7 @@ resource "google_cloud_run_v2_service" "gateway" { } dynamic "env" { - for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env, local.metrics_env_kv, local.gateway_pool_env) + for_each = local.gateway_env_kv content { name = env.value.name value = env.value.value @@ -243,7 +278,7 @@ resource "google_cloud_run_v2_service" "gateway" { } dynamic "env" { - for_each = concat(local.shared_env_secrets, local.otel_env_secrets, local.billing_metrics_env_secrets, local.gateway_extra_secret_kv) + for_each = local.gateway_env_secrets content { name = env.value.name value_source { @@ -357,6 +392,52 @@ resource "google_cloud_run_v2_service" "gateway" { } } + dynamic "containers" { + for_each = var.collector_enabled ? [1] : [] + content { + name = "spend-collector" + image = local.gateway_image + command = ["sh", "-c"] + args = [local.collector_args] + + resources { + limits = { + cpu = var.collector_cpu + memory = var.collector_memory + } + } + + dynamic "env" { + for_each = local.collector_env_kv_all + content { + name = env.value.name + value = env.value.value + } + } + + dynamic "env" { + for_each = local.collector_env_secrets + content { + name = env.value.name + value_source { + secret_key_ref { + secret = env.value.secret + version = env.value.version + } + } + } + } + + dynamic "volume_mounts" { + for_each = local.proxy_config_enabled ? [1] : [] + content { + name = local.proxy_config_volume + mount_path = local.proxy_config_mount_path + } + } + } + } + dynamic "volumes" { for_each = local.proxy_config_enabled ? [1] : [] content { diff --git a/terraform/litellm/gcp/tests/collector.tftest.hcl b/terraform/litellm/gcp/tests/collector.tftest.hcl new file mode 100644 index 00000000000..7a5ea781acb --- /dev/null +++ b/terraform/litellm/gcp/tests/collector.tftest.hcl @@ -0,0 +1,165 @@ +# Plan-only coverage for the opt-in collector sidecar on the gateway Cloud +# Run service. `mock_provider` keeps this offline: no GCP credentials, no API +# calls. Run from terraform/litellm/gcp with `terraform test`. + +mock_provider "google" {} +mock_provider "google-beta" {} +mock_provider "random" {} + +variables { + project_id = "acme-test" + region = "us-central1" + tenant = "acme" + env = "test" + allow_plaintext_lb = true +} + +run "disabled_by_default_leaves_the_service_untouched" { + command = plan + + assert { + condition = [for c in google_cloud_run_v2_service.gateway[0].template[0].containers : c.name] == ["gateway"] + error_message = "The gateway service must stay single-container unless collector_enabled is set." + } + + assert { + condition = !anytrue([ + for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : startswith(e.name, "LITELLM_COLLECTOR_") + ]) + error_message = "No LITELLM_COLLECTOR_* env may reach the gateway while the sidecar is disabled." + } +} + +run "enabled_adds_a_sidecar_that_shares_the_gateway_transport" { + command = plan + + variables { + collector_enabled = true + collector_port = 4321 + collector_buffer_size = 250 + collector_on_unavailable = "drop" + collector_cpu = "500m" + collector_memory = "1Gi" + gateway_extra_env = { OPENAI_API_BASE = "https://example.invalid" } + gateway_extra_secrets = { OPENAI_API_KEY = "projects/acme-test/secrets/openai-api-key" } + } + + assert { + condition = [for c in google_cloud_run_v2_service.gateway[0].template[0].containers : c.name] == ["gateway", "spend-collector"] + error_message = "Enabling the sidecar must append a spend-collector container after the gateway container." + } + + assert { + condition = alltrue([ + for c in google_cloud_run_v2_service.gateway[0].template[0].containers : ( + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_ENABLED"] == "true" && + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_ADDRESS"] == "tcp://127.0.0.1:4321" && + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_BUFFER_SIZE"] == "250" && + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_ON_UNAVAILABLE"] == "drop" && + { for e in c.env : e.name => e.value }["LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS"] == "10" + ) + ]) + error_message = "Gateway and sidecar must agree on the loopback address and the collector knobs." + } + + assert { + condition = ( + google_cloud_run_v2_service.gateway[0].template[0].containers[1].image == local.gateway_image && + google_cloud_run_v2_service.gateway[0].template[0].containers[1].command == tolist(["sh", "-c"]) && + endswith(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], " && exec python -m litellm.proxy.collector") && + strcontains(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], "export DATABASE_URL=") && + strcontains(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], "REDIS_SSL_CA_CERTS") && + { for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name => e.value }["LITELLM_JOB_ROLE"] == "collector" + ) + error_message = "The sidecar must run litellm.proxy.collector from the gateway image with the same Redis CA + DATABASE_URL bootstrap as the gateway." + } + + assert { + condition = ( + length(google_cloud_run_v2_service.gateway[0].template[0].containers[1].ports) == 0 && + google_cloud_run_v2_service.gateway[0].template[0].containers[1].resources[0].limits.cpu == "500m" && + google_cloud_run_v2_service.gateway[0].template[0].containers[1].resources[0].limits.memory == "1Gi" + ) + error_message = "The sidecar must not claim the ingress port and must carry its own resource limits." + } + + assert { + condition = ( + { for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name => e.value }["OPENAI_API_BASE"] == "https://example.invalid" && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name], "DATABASE_HOST") && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name], "REDIS_HOST") && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name if length(e.value_source) > 0], "LITELLM_MASTER_KEY") && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name if length(e.value_source) > 0], "DATABASE_PASSWORD") && + contains([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name if length(e.value_source) > 0], "OPENAI_API_KEY") + ) + error_message = "The sidecar must receive the gateway's database, Redis, and Secret Manager env plus gateway_extra_env / gateway_extra_secrets." + } +} + +run "coexists_with_the_metrics_sidecars" { + command = plan + + variables { + collector_enabled = true + gateway_metrics_port = 4001 + } + + assert { + condition = [for c in google_cloud_run_v2_service.gateway[0].template[0].containers : c.name] == ["gateway", "metrics", "collector", "spend-collector"] + error_message = "The spend collector must keep its own container name next to the GMP metrics collector." + } + + assert { + condition = ( + { for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name => e.value }["PROMETHEUS_MULTIPROC_DIR"] == local.metrics_multiproc_dir && + { for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e.name => e.value }["LITELLM_COLLECTOR_ENABLED"] == "true" + ) + error_message = "The gateway container must keep both the metrics and the collector env when both sidecars are on." + } +} + +run "sidecars_must_not_share_a_loopback_port" { + command = plan + + variables { + collector_enabled = true + collector_port = 4001 + gateway_metrics_port = 4001 + } + + expect_failures = [ + google_cloud_run_v2_service.gateway, + ] +} + +run "collector_cannot_take_the_metrics_sidecar_health_port" { + command = plan + + variables { + collector_enabled = true + collector_port = 13133 + } + + expect_failures = [ + var.collector_port, + ] +} + +run "proxy_config_is_mounted_into_the_sidecar_too" { + command = plan + + variables { + collector_enabled = true + proxy_config = { model_list = [] } + } + + assert { + condition = alltrue([ + for c in google_cloud_run_v2_service.gateway[0].template[0].containers : ( + [for m in c.volume_mounts : m.name] == [local.proxy_config_volume] && + contains([for e in c.env : e.name], "CONFIG_FILE_PATH") + ) + ]) + error_message = "Both containers must mount the proxy-config GCS volume and point CONFIG_FILE_PATH at it." + } +} diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index f298a0431c0..412f919ab89 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -656,3 +656,73 @@ variable "billing_metrics_ca_cert_pem" { default = "" sensitive = true } + +# ---------- Collector sidecar ---------- +# +# Opt-in offload of spend tracking from the gateway's uvicorn workers to a +# `python -m litellm.proxy.collector` sidecar container in the same Cloud Run +# instance (helm's `gateway.collector`, mirrors the AWS stack). Containers +# in one instance share localhost, so the sidecar listens on loopback TCP. +# Disabled (the default) adds nothing to the service. + +variable "collector_enabled" { + description = "Run the collector sidecar next to the gateway container and have the gateway ship spend events to it (sets LITELLM_COLLECTOR_ENABLED=true on both). The sidecar shares the instance's request-based CPU allocation, so pair it with a non-zero gateway_min_instances if spend must keep flowing between requests." + type = bool + default = false +} + +variable "collector_port" { + description = "Loopback TCP port the sidecar listens on (LITELLM_COLLECTOR_ADDRESS=tcp://127.0.0.1:)." + type = number + default = 4010 + + validation { + condition = var.collector_port >= 1024 && var.collector_port <= 65535 && !contains([4000, 13133], var.collector_port) + error_message = "collector_port must be in 1024-65535 and not 4000 (the gateway port) or 13133 (the metrics sidecar health port)." + } +} + +variable "collector_cpu" { + description = "Cloud Run CPU limit for the sidecar container, on top of gateway_cpu. Matches helm's collector.resources.limits.cpu." + type = string + default = "1000m" +} + +variable "collector_memory" { + description = "Cloud Run memory limit for the sidecar container, on top of gateway_memory. Matches helm's collector.resources.limits.memory." + type = string + default = "2Gi" +} + +variable "collector_buffer_size" { + description = "Per-worker in-memory queue of spend events waiting to be shipped to the sidecar (LITELLM_COLLECTOR_BUFFER_SIZE)." + type = number + default = 1000 + + validation { + condition = var.collector_buffer_size >= 1 + error_message = "collector_buffer_size must be >= 1." + } +} + +variable "collector_on_unavailable" { + description = "What the gateway does with spend events when the sidecar is unreachable or the buffer is full (LITELLM_COLLECTOR_ON_UNAVAILABLE): `fallback` runs the pipeline in-process, `drop` discards them." + type = string + default = "fallback" + + validation { + condition = contains(["fallback", "drop"], var.collector_on_unavailable) + error_message = "collector_on_unavailable must be one of: fallback, drop." + } +} + +variable "collector_drain_timeout_seconds" { + description = "Seconds a gateway worker waits on shutdown for its buffered spend events to reach the sidecar (LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS)." + type = number + default = 10 + + validation { + condition = var.collector_drain_timeout_seconds > 0 + error_message = "collector_drain_timeout_seconds must be > 0." + } +} diff --git a/tests/test_gateway/test_launch.py b/tests/test_gateway/test_launch.py index 6039aaafc2b..a783ce6ac7e 100644 --- a/tests/test_gateway/test_launch.py +++ b/tests/test_gateway/test_launch.py @@ -3,6 +3,7 @@ import socket import sys import textwrap import urllib.parse +from collections.abc import Iterator from pathlib import Path from typing import Final, cast from unittest.mock import MagicMock, patch @@ -66,7 +67,7 @@ def _query(url: str) -> dict[str, str]: @pytest.fixture -def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: +def password_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[dict[str, str]]: for var in ( "DATABASE_URL", "IAM_TOKEN_DB_AUTH", @@ -78,7 +79,8 @@ def password_env(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: monkeypatch.delenv(var) for var, value in DB_ENV.items(): monkeypatch.setenv(var, value) - return dict(DB_ENV) + yield dict(DB_ENV) + os.environ.pop("DATABASE_URL", None) def _minted_iam_token(token: str): diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py index 8e0bc200012..071b99850f6 100644 --- a/tests/test_litellm/caching/test_caching_handler.py +++ b/tests/test_litellm/caching/test_caching_handler.py @@ -658,3 +658,38 @@ def test_async_cache_write_completes_when_asyncio_run_closes_the_loop(monkeypatc asyncio.run(_short_lived_script()) assert len(writes) == 1 + + +@pytest.mark.asyncio +async def test_cache_hit_records_the_looked_up_key_as_the_preset_cache_key(monkeypatch): + """The spend log for a cache hit must reuse the key the lookup already computed instead of hashing again.""" + import litellm + from litellm.caching.caching import Cache + from litellm.types.utils import CallTypes + + async def acompletion(**kwargs): + return None + + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + kwargs = {"model": "gpt-5.4", "messages": [{"role": "user", "content": "hello"}], "caching": True} + await litellm.cache.async_add_cache( + litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "hi"}}]), **kwargs + ) + handler = LLMCachingHandler(original_function=acompletion, request_kwargs=kwargs, start_time=datetime.now()) + logging_obj = _build_logging_obj(CallTypes.acompletion.value, stream=False) + logging_obj.async_success_handler = AsyncMock() + + hit = await handler._async_get_cache( + model="gpt-5.4", + original_function=acompletion, + logging_obj=logging_obj, + start_time=datetime.now(), + call_type=CallTypes.acompletion.value, + kwargs=kwargs, + args=(), + ) + + assert hit is not None and hit.cached_result is not None + assert handler.preset_cache_key is not None + assert logging_obj.litellm_params["preset_cache_key"] == handler.preset_cache_key + assert hit.cached_result._hidden_params["cache_key"] == handler.preset_cache_key diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index eff892f2d80..c96cfc3ee4a 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -5,14 +5,20 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth +from litellm.proxy.collector import SpendEventConsumer +from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.proxy.hooks.proxy_track_cost_callback import ( _get_budget_reservation_from_metadata, _ProxyDBLogger, _should_track_cost_callback, _update_database_and_spend_counters, + run_spend_event, ) -from litellm.types.utils import CallTypes, Usage +from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event +from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, Usage @pytest.mark.asyncio @@ -2096,3 +2102,248 @@ async def test_spend_counters_keep_every_granted_group_when_the_deployment_is_un ) assert charged == ("premium", "tier0") + + +def _offload_kwargs() -> dict: + big_prompt = "x" * 10_000 + reservation = {"reserved_cost": 0.5, "entries": [{"counter_key": "key:hash-1", "reserved_cost": 0.5}]} + return { + "litellm_call_id": "call-1", + "call_type": "acompletion", + "model": "gpt-4o", + "custom_llm_provider": "openai", + "stream": False, + "cache_hit": None, + "response_cost": 0.0125, + "completion_start_time": datetime(2026, 1, 1, 0, 0, 1), + "messages": [{"role": "user", "content": big_prompt}], + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "litellm_params": { + "api_base": "https://api.openai.com", + "preset_cache_key": None, + "proxy_server_request": {"body": {"messages": [{"role": "user", "content": big_prompt}]}}, + "metadata": { + "user_api_key": "hash-1", + "user_api_key_hash": "hash-1", + "user_api_key_alias": "alias-1", + "user_api_key_user_id": "user-1", + "user_api_key_team_id": "team-1", + "user_api_key_org_id": "org-1", + "user_api_key_end_user_id": "end-user-1", + "user_api_key_auth": UserAPIKeyAuth(api_key="hash-1", budget_reservation=reservation), + "model_group": "gpt-4o", + "model_info": {"id": "deployment-1"}, + "tags": ["tag-a"], + }, + }, + "standard_logging_object": { + "id": "chatcmpl-1", + "trace_id": "trace-1", + "response_cost": 0.0125, + "model": "gpt-4o-2024-08-06", + "model_id": "deployment-1", + "model_group": "gpt-4o", + "api_base": "https://api.openai.com", + "custom_llm_provider": "openai", + "prompt_tokens": 5000, + "completion_tokens": 4000, + "total_tokens": 9000, + "request_tags": ["tag-a"], + "request_model_access_groups": ["premium"], + "messages": [{"role": "user", "content": big_prompt}], + "response": {"choices": [{"message": {"content": "y" * 10_000}}]}, + "model_parameters": {"temperature": 0.1}, + "metadata": { + "user_api_key_hash": "hash-1", + "user_api_key_end_user_id": "end-user-1", + "usage_object": {"prompt_tokens": 5000, "completion_tokens": 4000, "total_tokens": 9000}, + }, + "hidden_params": {"litellm_overhead_time_ms": 3}, + "model_map_information": {}, + "cost_breakdown": {"input_cost": 0.0125, "output_cost": 0.0}, + }, + } + + +def _offload_response() -> ModelResponse: + return ModelResponse( + id="chatcmpl-1", + model="gpt-4o-2024-08-06", + choices=[ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call-1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + "finish_reason": "tool_calls", + } + ], + usage=Usage(prompt_tokens=5000, completion_tokens=4000, total_tokens=9000), + ) + + +class _RecordingHandler: + def __init__(self) -> None: + self.lines: list[bytes] = [] # mutable-ok: test double records the events the sidecar received + + async def __call__(self, line: bytes) -> None: + self.lines.append(line) + + +async def _no_fallback(line: bytes) -> None: + raise AssertionError("the sidecar was reachable, nothing should fall back") + + +@pytest.mark.asyncio +async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_skips_the_pipeline(tmp_path): + handler = _RecordingHandler() + consumer = SpendEventConsumer(handler) + address = UnixAddress(path=str(tmp_path / "spend.sock")) + server = await consumer.serve(address) + producer = SpendEventProducer( + address=address, on_unavailable="fallback", buffer_size=10, connect_timeout=1.0, fallback=_no_fallback + ) + logger = _ProxyDBLogger(producer) + + with ( + patch( # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_proxy_logging, + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock + ) as counters, + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + await logger.async_log_success_event(_offload_kwargs(), _offload_response(), datetime.now(), datetime.now()) + await producer.close(drain_timeout=5.0) + server.close() + assert await consumer.drain(timeout=5.0) == 0 + + mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() + counters.assert_not_awaited() + assert producer.stats().sent == 1 + assert len(handler.lines) == 1 + assert len(handler.lines[0]) < 4_000 + event = decode_spend_event(handler.lines[0]) + assert not isinstance(event, SpendEventDecodeError) + assert event.litellm_params["metadata"]["user_api_key_team_id"] == "team-1" + assert event.response_cost == 0.0125 + + +@pytest.mark.asyncio +async def test_async_log_success_event_keeps_batch_retrieves_in_process(): + producer = SpendEventProducer( + address=UnixAddress(path="/nonexistent/spend.sock"), + on_unavailable="drop", + buffer_size=10, + connect_timeout=1.0, + fallback=_no_fallback, + ) + logger = _ProxyDBLogger(producer) + kwargs = {**_offload_kwargs(), "call_type": CallTypes.aretrieve_batch.value} + completed_batch = LiteLLMBatch( + id="batch_abc", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-in", + output_file_id="file-out", + object="batch", + status="completed", + ) + + with ( + patch( # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_proxy_logging, + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock + ), + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock + ), + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True) + mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock() + await logger.async_log_success_event(kwargs, completed_batch, datetime.now(), datetime.now()) + + mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once() + assert producer.stats().queued == 0 + + +async def _spend_row_written_by(run) -> tuple[SpendLogsPayload, dict, tuple[str, ...]]: + with ( + patch( # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_proxy_logging, + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock + ) as counters, + patch( # test-quality-ok: same function-body import, no injection seam + "litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock + ), + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True) + mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock() + await run() + mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once() + written = mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs + counters.assert_awaited_once() + counted = dict(counters.await_args.kwargs) + + row = get_logging_payload( + kwargs=written["kwargs"], + response_obj=written["completion_response"], + start_time=written["start_time"], + end_time=written["end_time"], + ) + return row, counted, response_tool_call_names(written["completion_response"]) + + +@pytest.mark.asyncio +async def test_sidecar_writes_the_same_spend_row_and_counters_as_the_in_process_path(): + start_time = datetime(2026, 1, 1, 0, 0, 0) + end_time = datetime(2026, 1, 1, 0, 0, 2) + + async def in_process() -> None: + await _ProxyDBLogger().async_log_success_event(_offload_kwargs(), _offload_response(), start_time, end_time) + + async def via_sidecar() -> None: + line = build_spend_event(_offload_kwargs(), _offload_response(), start_time, end_time, store_bodies=False) + assert isinstance(line, bytes) + await run_spend_event(line) + + in_process_row, in_process_counters, in_process_tools = await _spend_row_written_by(in_process) + sidecar_row, sidecar_counters, sidecar_tools = await _spend_row_written_by(via_sidecar) + + assert sidecar_row == in_process_row + assert in_process_row["spend"] == 0.0125 + assert in_process_row["team_id"] == "team-1" + assert in_process_row["end_user"] == "end-user-1" + assert in_process_row["total_tokens"] == 9000 + assert in_process_row["model_id"] == "deployment-1" + assert in_process_row["request_tags"] == '["tag-a"]' + assert in_process_row["messages"] == "{}" + assert in_process_row["response"] == "{}" + assert sidecar_counters == in_process_counters + assert in_process_counters["token"] == "hash-1" + assert in_process_counters["response_cost"] == 0.0125 + assert in_process_counters["budget_reservation"]["reserved_cost"] == 0.5 + assert in_process_counters["model_access_groups"] == ("premium",) + assert sidecar_tools == in_process_tools == ("get_weather",) + + +@pytest.mark.asyncio +async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a discarded event has no observable output other than the DB writer never being reached + with ( + patch( # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_proxy_logging + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + await run_spend_event(b"garbage\n") + mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index dafe686c4f2..deb7289d2d1 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -253,6 +253,38 @@ async def test_flush_spend_logs_queue_on_shutdown_swallows_drain_errors(monkeypa await ps._flush_spend_logs_queue_on_shutdown() +@pytest.mark.asyncio +async def test_flush_spend_counters_on_shutdown_commits_buffered_spend(monkeypatch): + fake_prisma = MagicMock() + monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + commit = AsyncMock() + monkeypatch.setattr(ps.proxy_logging_obj.db_spend_update_writer, "db_update_spend_transaction_handler", commit) + + await ps.flush_spend_counters_on_shutdown() + + observed = { + "commit_calls": commit.await_count, + "commit_prisma": commit.await_args.kwargs["prisma_client"] is fake_prisma, + "commit_proxy_logging": commit.await_args.kwargs["proxy_logging_obj"] is ps.proxy_logging_obj, + } + assert observed == {"commit_calls": 1, "commit_prisma": True, "commit_proxy_logging": True} + + +@pytest.mark.asyncio +async def test_flush_spend_counters_on_shutdown_logs_and_swallows_commit_errors(monkeypatch, caplog): + monkeypatch.setattr(ps, "prisma_client", MagicMock(), raising=False) + monkeypatch.setattr( + ps.proxy_logging_obj.db_spend_update_writer, + "db_update_spend_transaction_handler", + AsyncMock(side_effect=RuntimeError("db gone")), + ) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + await ps.flush_spend_counters_on_shutdown() + + assert "Error flushing spend counters on shutdown: db gone" in caplog.text + + # --------------------------------------------------------------------------- # _initialize_shared_aiohttp_session # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_event.py b/tests/test_litellm/proxy/spend_tracking/test_spend_event.py new file mode 100644 index 00000000000..ff449235582 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_event.py @@ -0,0 +1,213 @@ +import json +from datetime import datetime +from typing import Final + +import pytest + +import litellm +from litellm.caching.caching import Cache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.spend_tracking.spend_event import ( + CACHE_OFF_KEY, + SpendEventBuildError, + SpendEventDecodeError, + build_spend_event, + decode_spend_event, + is_offloadable_success, + spend_event_callback_args, +) +from litellm.types.utils import LiteLLMBatch, ModelResponse, Usage + +_BIG_PROMPT: Final = "x" * 20_000 +_RESERVATION: Final = { + "reserved_cost": 0.5, + "entries": [{"counter_key": "key:hash", "reserved_cost": 0.5}], + "finalized": False, + "input_cost": 0.1, + "input_tokens": 5000, +} + + +def _response(tool_name: str | None = None) -> ModelResponse: + tool_calls: Final = ( + [{"id": "call-1", "type": "function", "function": {"name": tool_name, "arguments": "{}"}}] + if tool_name is not None + else None + ) + return ModelResponse( + id="chatcmpl-1", + model="gpt-4o-2024-08-06", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "y" * 20_000, "tool_calls": tool_calls}, + "finish_reason": "tool_calls" if tool_name else "stop", + } + ], + usage=Usage(prompt_tokens=5000, completion_tokens=4000, total_tokens=9000), + ) + + +def _success_kwargs(preset_cache_key: str | None = "preset-key") -> dict: + return { + "litellm_call_id": "call-1", + "call_type": "acompletion", + "model": "gpt-4o", + "custom_llm_provider": "openai", + "stream": False, + "cache_hit": None, + "response_cost": 0.0125, + "completion_start_time": datetime(2026, 1, 1, 0, 0, 1), + "messages": [{"role": "user", "content": _BIG_PROMPT}], + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "litellm_params": { + "api_base": "https://api.openai.com", + "preset_cache_key": preset_cache_key, + "proxy_server_request": {"body": {"messages": [{"role": "user", "content": _BIG_PROMPT}]}}, + "metadata": { + "user_api_key": "hash-1", + "user_api_key_user_id": "user-1", + "user_api_key_team_id": "team-1", + "user_api_key_org_id": "org-1", + "user_api_key_end_user_id": "end-user-1", + "user_api_key_auth": UserAPIKeyAuth(api_key="hash-1", budget_reservation=dict(_RESERVATION)), + "model_group": "gpt-4o", + "model_info": {"id": "deployment-1"}, + "tags": ["tag-a"], + "litellm_parent_otel_span": object(), + }, + }, + "standard_logging_object": { + "response_cost": 0.0125, + "model": "gpt-4o-2024-08-06", + "model_id": "deployment-1", + "request_tags": ["tag-a"], + "request_model_access_groups": ["premium"], + "messages": [{"role": "user", "content": _BIG_PROMPT}], + "response": {"choices": [{"message": {"content": "y" * 20_000}}]}, + "model_parameters": {"temperature": 0.1}, + "metadata": {"user_api_key_hash": "hash-1", "usage_object": {"prompt_tokens": 5000}}, + "hidden_params": {"litellm_overhead_time_ms": 3}, + "model_map_information": {}, + }, + } + + +def _build(kwargs: dict, response: object, store_bodies: bool = False) -> bytes: + line: Final = build_spend_event( + kwargs, response, datetime(2026, 1, 1), datetime(2026, 1, 1, 0, 0, 2), store_bodies=store_bodies + ) + assert isinstance(line, bytes) + return line + + +def test_event_is_compact_and_omits_bodies_by_default(): + line: Final = _build(_success_kwargs(), _response(tool_name="get_weather")) + assert line.endswith(b"\n") + assert len(line) < 4_000 + assert _BIG_PROMPT.encode() not in line + assert b"yyyy" not in line + decoded: Final = json.loads(line) + assert "messages" not in decoded["standard_logging_object"] + assert "response" not in decoded["standard_logging_object"] + assert decoded["litellm_params"]["proxy_server_request"] is None + + +def test_event_carries_bodies_when_spend_logs_store_them(): + line: Final = _build(_success_kwargs(), _response(), store_bodies=True) + decoded: Final = json.loads(line) + assert decoded["standard_logging_object"]["messages"][0]["content"] == _BIG_PROMPT + assert decoded["standard_logging_object"]["response"]["choices"][0]["message"]["content"] == "y" * 20_000 + assert decoded["litellm_params"]["proxy_server_request"]["body"]["messages"][0]["content"] == _BIG_PROMPT + + +def test_round_trip_preserves_identity_usage_reservation_and_tools(): + line: Final = _build(_success_kwargs(), _response(tool_name="get_weather")) + event: Final = decode_spend_event(line) + assert not isinstance(event, SpendEventDecodeError) + args: Final = spend_event_callback_args(event) + + metadata: Final = args.kwargs["litellm_params"]["metadata"] + assert metadata is not None + assert (metadata["user_api_key"], metadata["user_api_key_team_id"], metadata["user_api_key_org_id"]) == ( + "hash-1", + "team-1", + "org-1", + ) + assert metadata["user_api_key_budget_reservation"] == _RESERVATION + assert "user_api_key_auth" not in metadata + assert "litellm_parent_otel_span" not in metadata + assert args.kwargs["standard_logging_object"]["request_model_access_groups"] == ["premium"] + assert args.kwargs["standard_logging_object"]["response_cost"] == 0.0125 + assert args.kwargs["tools"] == ({"type": "function", "function": {"name": "get_weather"}},) + assert args.kwargs["completion_start_time"] == datetime(2026, 1, 1, 0, 0, 1) + assert (args.start_time, args.end_time) == (datetime(2026, 1, 1), datetime(2026, 1, 1, 0, 0, 2)) + assert args.response_obj is not None + assert args.response_obj["id"] == "chatcmpl-1" + assert args.response_obj["usage"]["prompt_tokens"] == 5000 + assert args.response_obj["usage"]["completion_tokens"] == 4000 + tool_calls: Final = args.response_obj["choices"][0]["message"]["tool_calls"] + assert [call["function"]["name"] for call in tool_calls] == ["get_weather"] + assert "complete_streaming_response" not in args.kwargs + + +def test_streaming_event_reconstructs_complete_streaming_response(): + kwargs: Final = {**_success_kwargs(), "stream": True, "complete_streaming_response": _response()} + event: Final = decode_spend_event(_build(kwargs, _response())) + assert not isinstance(event, SpendEventDecodeError) + args: Final = spend_event_callback_args(event) + assert args.kwargs["stream"] is True + assert args.kwargs["complete_streaming_response"] == args.response_obj + + +class _HashingCache(Cache): + def __init__(self) -> None: + pass + + def get_cache_key(self, **kwargs) -> str: + raise AssertionError("the fast path must not hash the request body") + + +@pytest.mark.parametrize( + ("cache", "preset", "expected"), + [ + (None, "preset-key", CACHE_OFF_KEY), + (_HashingCache(), "preset-key", "preset-key"), + (_HashingCache(), None, None), + ], +) +def test_event_reuses_preset_cache_key_and_never_hashes(monkeypatch, cache, preset, expected): + monkeypatch.setattr(litellm, "cache", cache) + decoded: Final = json.loads(_build(_success_kwargs(preset_cache_key=preset), _response())) + assert decoded["litellm_params"]["preset_cache_key"] == expected + + +def test_unbuildable_kwargs_fall_back_to_in_process_tracking(): + kwargs: Final = {**_success_kwargs(), "response_cost": "not-a-number"} + assert isinstance( + build_spend_event(kwargs, _response(), datetime.now(), datetime.now(), False), SpendEventBuildError + ) + + +def test_undecodable_line_is_an_error_value(): + assert isinstance(decode_spend_event(b'{"version": 2}\n'), SpendEventDecodeError) + assert isinstance(decode_spend_event(b"not json\n"), SpendEventDecodeError) + + +def test_batch_retrieves_stay_in_process(): + assert is_offloadable_success(_response()) is True + assert is_offloadable_success(None) is True + assert ( + is_offloadable_success( + LiteLLMBatch( + id="batch-1", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="f", + object="batch", + status="completed", + ) + ) + is False + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_event_producer.py b/tests/test_litellm/proxy/spend_tracking/test_spend_event_producer.py new file mode 100644 index 00000000000..adfb1251d1c --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_event_producer.py @@ -0,0 +1,359 @@ +import asyncio +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Final + +import pytest +import uvloop + +from litellm.proxy.spend_tracking.spend_event_producer import ( + AddressError, + CollectorAddress, + CollectorSettings, + SpendEventProducer, + TcpAddress, + UnixAddress, + build_spend_event_producer, + open_collector_connection, + parse_collector_address, +) + + +class _Sidecar: + """A unix-socket server that records every line it receives, standing in for the collector.""" + + def __init__(self, path: Path, reads: bool = True, limit: int = 2**16) -> None: + self.path = path + self.reads = reads + self.limit = limit + self.lines: list[bytes] = [] # mutable-ok: test double records what the producer sent + self._server: asyncio.Server | None = None + self._stopped = asyncio.Event() + self._connections: list[asyncio.StreamWriter] = [] # mutable-ok: test double tracks peers to hang up on + + async def __aenter__(self) -> "_Sidecar": + self._server = await asyncio.start_unix_server(self._on_connection, path=str(self.path), limit=self.limit) + return self + + async def __aexit__(self, *exc: object) -> None: + self._stopped.set() + await self.hang_up() + + async def hang_up(self) -> None: + """Exit the way a stopped sidecar does: stop listening and close every producer connection.""" + assert self._server is not None + self._server.close() + for connection in self._connections: + connection.close() + await connection.wait_closed() + await self._server.wait_closed() + + async def _on_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self._connections.append(writer) + if not self.reads: + await self._stopped.wait() + return + while line := await reader.readline(): + self.lines.append(line) + writer.close() + + +class _CrashingSidecar(_Sidecar): + """Bills a few lines, then dies mid-stream with the producer's backlog still queued behind them.""" + + def __init__(self, path: Path, lines_before_crash: int) -> None: + super().__init__(path, limit=2**20) + self._lines_before_crash = lines_before_crash + + async def _on_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self._connections.append(writer) + for _ in range(self._lines_before_crash): + self.lines.append(await reader.readline()) + writer.transport.abort() + + +class _Fallback: + def __init__(self) -> None: + self.lines: list[bytes] = [] # mutable-ok: test double records what fell back to in-process + + async def __call__(self, line: bytes) -> None: + self.lines.append(line) + + +class _GatedFallback(_Fallback): + """A fallback that blocks, like a slow database write, until the test releases it.""" + + def __init__(self) -> None: + super().__init__() + self.started = asyncio.Event() + self.release = asyncio.Event() + + async def __call__(self, line: bytes) -> None: + self.started.set() + await self.release.wait() + await super().__call__(line) + + +class _StalledDrainWriter(asyncio.StreamWriter): + """Hands bytes to the real transport but never wakes ``drain()``: the loop iteration between a flush + completing and the writer task resuming, frozen in place.""" + + def __init__(self, real: asyncio.StreamWriter, reader: asyncio.StreamReader) -> None: + super().__init__(real.transport, real.transport.get_protocol(), reader, asyncio.get_running_loop()) + self._real_writer_whose_finalizer_would_close_the_transport = real + + async def drain(self) -> None: + await asyncio.Event().wait() + + +async def _open_with_stalled_drain( + address: CollectorAddress, timeout: float +) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + reader, writer = await open_collector_connection(address, timeout) + return reader, _StalledDrainWriter(writer, reader) + + +def _producer( + path: Path, + fallback: _Fallback, + on_unavailable="fallback", + buffer_size: int = 100, + open_connection: Callable[ + [CollectorAddress, float], Awaitable[tuple[asyncio.StreamReader, asyncio.StreamWriter]] + ] = open_collector_connection, +) -> SpendEventProducer: + return SpendEventProducer( + address=UnixAddress(path=str(path)), + on_unavailable=on_unavailable, + buffer_size=buffer_size, + connect_timeout=1.0, + fallback=fallback, + open_connection=open_connection, + ) + + +def test_parse_collector_address(): + assert parse_collector_address("unix:///var/run/litellm/collector.sock") == UnixAddress( + path="/var/run/litellm/collector.sock" + ) + assert parse_collector_address("tcp://127.0.0.1:4100") == TcpAddress(host="127.0.0.1", port=4100) + assert parse_collector_address("tcp://localhost:4100") == TcpAddress(host="localhost", port=4100) + assert parse_collector_address("tcp://[::1]:4100") == TcpAddress(host="::1", port=4100) + assert isinstance(parse_collector_address("redis://localhost:6379"), AddressError) + assert isinstance(parse_collector_address("tcp://127.0.0.1"), AddressError) + + +@pytest.mark.parametrize("address", ["tcp://0.0.0.0:4100", "tcp://10.0.0.5:4100", "tcp://collector.svc:4100"]) +def test_tcp_address_outside_loopback_is_refused(address: str): + """The socket has no authentication, so anything reachable from outside the pod would accept forged spend.""" + error: Final = parse_collector_address(address) + assert isinstance(error, AddressError) + assert "loopback" in error.reason + assert build_spend_event_producer(CollectorSettings(enabled=True, address=address), _Fallback()) is None + + +def test_gateway_produces_only_when_enabled_and_not_the_sidecar_itself(): + fallback: Final = _Fallback() + assert build_spend_event_producer(CollectorSettings(enabled=False), fallback) is None + assert build_spend_event_producer(CollectorSettings(enabled=True, job_role="collector"), fallback) is None + assert build_spend_event_producer(CollectorSettings(enabled=True, address="redis://x"), fallback) is None + assert isinstance(build_spend_event_producer(CollectorSettings(enabled=True), fallback), SpendEventProducer) + + +def test_settings_read_the_documented_env(monkeypatch): + monkeypatch.setenv("LITELLM_COLLECTOR_ENABLED", "true") + monkeypatch.setenv("LITELLM_COLLECTOR_ADDRESS", "tcp://127.0.0.1:4100") + monkeypatch.setenv("LITELLM_COLLECTOR_BUFFER_SIZE", "50") + monkeypatch.setenv("LITELLM_COLLECTOR_ON_UNAVAILABLE", "drop") + monkeypatch.setenv("LITELLM_JOB_ROLE", "collector") + settings: Final = CollectorSettings() + assert (settings.enabled, settings.address, settings.buffer_size, settings.on_unavailable) == ( + True, + "tcp://127.0.0.1:4100", + 50, + "drop", + ) + assert settings.produces is False + + +@pytest.mark.asyncio +async def test_events_reach_the_sidecar_once_and_in_order(tmp_path: Path): + fallback: Final = _Fallback() + async with _Sidecar(tmp_path / "spend.sock") as sidecar: + producer: Final = _producer(sidecar.path, fallback) + outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(20)] + await producer.close(drain_timeout=5.0) + await asyncio.sleep(0.05) + + assert outcomes == ["queued"] * 20 + assert sidecar.lines == [f"event-{i}\n".encode() for i in range(20)] + assert fallback.lines == [] + stats: Final = producer.stats() + assert (stats.queued, stats.sent, stats.fallback, stats.dropped) == (20, 20, 0, 0) + + +@pytest.mark.asyncio +async def test_unreachable_sidecar_falls_back_in_process_and_backs_off(tmp_path: Path): + fallback: Final = _Fallback() + producer: Final = _producer(tmp_path / "missing.sock", fallback) + first: Final = await producer.publish(b"event-1\n") + await asyncio.sleep(0.05) + second: Final = await producer.publish(b"event-2\n") + await producer.close(drain_timeout=5.0) + + assert first == "queued" + assert second == "fallback" + assert fallback.lines == [b"event-1\n", b"event-2\n"] + stats: Final = producer.stats() + assert (stats.sent, stats.fallback, stats.dropped, stats.connected) == (0, 2, 0, False) + + +@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) +def test_sidecar_hang_up_falls_back_instead_of_losing_events( + tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] +): + async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]: + fallback: Final = _Fallback() + sidecar: Final = _Sidecar(tmp_path / "spend.sock") + async with sidecar: + producer: Final = _producer(sidecar.path, fallback) + await producer.publish(b"event-1\n") + await asyncio.sleep(0.05) + await sidecar.hang_up() + await asyncio.sleep(0.05) + await producer.publish(b"event-2\n") + await producer.close(drain_timeout=5.0) + stats: Final = producer.stats() + return sidecar.lines, fallback.lines, (stats.sent, stats.fallback, stats.dropped) + + with asyncio.Runner(loop_factory=loop_factory) as runner: + sidecar_lines, fallback_lines, counts = runner.run(scenario()) + + assert sidecar_lines == [b"event-1\n"] + assert fallback_lines == [b"event-2\n"] + assert counts == (1, 1, 0) + + +@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) +def test_mid_stream_crash_never_bills_an_event_on_both_sides( + tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] +): + """Events large enough to straddle the kernel buffer, a sidecar that reads some and then drops the socket: a + failed write may only fall back when the sidecar cannot have read the whole line.""" + events: Final = tuple(f"event-{i:03d}-".encode() + b"x" * 65536 + b"\n" for i in range(64)) + + async def scenario() -> tuple[list[bytes], list[bytes], tuple[int, int, int]]: + fallback: Final = _Fallback() + sidecar: Final = _CrashingSidecar(tmp_path / "spend.sock", lines_before_crash=3) + async with sidecar: + producer: Final = _producer(sidecar.path, fallback) + for event in events: + assert await producer.publish(event) == "queued" + await asyncio.sleep(0.2) + await producer.close(drain_timeout=5.0) + stats: Final = producer.stats() + return sidecar.lines, fallback.lines, (stats.sent, stats.fallback, stats.dropped) + + with asyncio.Runner(loop_factory=loop_factory) as runner: + sidecar_lines, fallback_lines, counts = runner.run(scenario()) + + assert sidecar_lines == list(events[:3]) + assert set(sidecar_lines).isdisjoint(fallback_lines) + assert len(fallback_lines) == len(set(fallback_lines)) + assert fallback_lines[-1] == events[-1] + assert counts[0] + counts[1] == len(events) and counts[2] == 0 + assert counts[0] >= len(sidecar_lines) + + +@pytest.mark.asyncio +async def test_drain_timeout_hands_the_in_flight_event_to_fallback(tmp_path: Path): + """A sidecar that stops reading leaves one event half-written; cancelling the writer must not lose it.""" + fallback: Final = _Fallback() + stuck: Final = b"x" * (4 * 1024 * 1024) + b"\n" + async with _Sidecar(tmp_path / "spend.sock", reads=False) as sidecar: + producer: Final = _producer(sidecar.path, fallback) + assert await producer.publish(stuck) == "queued" + await asyncio.sleep(0.1) + await producer.close(drain_timeout=0.2) + + assert fallback.lines == [stuck] + stats: Final = producer.stats() + assert (stats.sent, stats.fallback, stats.connected) == (0, 1, False) + + +@pytest.mark.asyncio +async def test_shutdown_lets_the_writer_finish_a_fallback_already_in_progress(tmp_path: Path): + """Cancelling the writer while it runs the pipeline in-process must neither lose nor repeat that event.""" + fallback: Final = _GatedFallback() + producer: Final = _producer(tmp_path / "missing.sock", fallback) + assert await producer.publish(b"event-1\n") == "queued" + await asyncio.wait_for(fallback.started.wait(), 5.0) + closing: Final = asyncio.ensure_future(producer.close(drain_timeout=0.05)) + await asyncio.sleep(0.2) + assert fallback.lines == [] + fallback.release.set() + await asyncio.wait_for(closing, 5.0) + + assert fallback.lines == [b"event-1\n"] + assert producer.stats().fallback == 1 + + +@pytest.mark.asyncio +async def test_shutdown_does_not_replay_an_event_the_kernel_already_took(tmp_path: Path): + """Cancelling a drain whose bytes already left the process must not run the event a second time in-process.""" + fallback: Final = _Fallback() + async with _Sidecar(tmp_path / "spend.sock") as sidecar: + producer: Final = _producer(sidecar.path, fallback, open_connection=_open_with_stalled_drain) + assert await producer.publish(b"event-1\n") == "queued" + await asyncio.sleep(0.1) + await producer.close(drain_timeout=0.2) + await asyncio.sleep(0.05) + + assert sidecar.lines == [b"event-1\n"] + assert fallback.lines == [] + stats: Final = producer.stats() + assert (stats.fallback, stats.dropped, stats.connected) == (0, 0, False) + + +@pytest.mark.asyncio +async def test_drop_policy_counts_instead_of_running_in_process(tmp_path: Path): + fallback: Final = _Fallback() + producer: Final = _producer(tmp_path / "missing.sock", fallback, on_unavailable="drop") + await producer.publish(b"event-1\n") + await producer.close(drain_timeout=5.0) + assert await producer.publish(b"event-2\n") == "dropped" + + assert fallback.lines == [] + assert producer.stats().dropped == 2 + + +@pytest.mark.asyncio +async def test_full_buffer_applies_the_unavailable_policy_immediately(tmp_path: Path): + fallback: Final = _Fallback() + async with _Sidecar(tmp_path / "spend.sock") as sidecar: + producer: Final = _producer(sidecar.path, fallback, buffer_size=2) + outcomes: Final = [await producer.publish(f"event-{i}\n".encode()) for i in range(3)] + await producer.close(drain_timeout=5.0) + await asyncio.sleep(0.05) + + assert outcomes == ["queued", "queued", "fallback"] + assert fallback.lines == [b"event-2\n"] + assert sidecar.lines == [b"event-0\n", b"event-1\n"] + + +@pytest.mark.asyncio +async def test_close_flushes_buffered_events_then_refuses_new_ones(tmp_path: Path): + fallback: Final = _Fallback() + async with _Sidecar(tmp_path / "spend.sock") as sidecar: + producer: Final = _producer(sidecar.path, fallback) + for i in range(50): + await producer.publish(f"event-{i}\n".encode()) + assert sidecar.lines == [] + await producer.close(drain_timeout=5.0) + await asyncio.sleep(0.05) + after_close: Final = await producer.publish(b"late\n") + + assert len(sidecar.lines) == 50 + assert after_close == "fallback" + assert fallback.lines == [b"late\n"] + assert producer.stats().connected is False diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index d84d63ce1a4..df113197ec6 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -38,9 +38,9 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_error_information_for_spend_logs, _sanitize_guardrail_information_for_spend_logs, _sanitize_request_body_for_spend_logs_payload, - _should_store_prompts_and_responses_in_spend_logs, get_logging_payload, get_spend_logs_id, + should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import hash_token from litellm.types.utils import ( @@ -107,6 +107,48 @@ def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_token assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123 +class _HashingCache(litellm.Cache): + def __init__(self) -> None: + pass + + def get_cache_key(self, **kwargs) -> str: + raise AssertionError("a preset cache key must be reused instead of hashing the request") + + +def _cache_key_in_spend_log(monkeypatch: pytest.MonkeyPatch, cache: litellm.Cache | None, preset: str | None) -> str: + monkeypatch.setattr(litellm, "cache", cache) + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "x" * 10_000}], + "litellm_params": {"metadata": {"user_api_key": "test-key"}, "preset_cache_key": preset}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + return payload["cache_key"] + + +def test_get_logging_payload_reuses_the_preset_cache_key_instead_of_hashing_the_body(monkeypatch): + assert _cache_key_in_spend_log(monkeypatch, _HashingCache(), "preset-key") == "preset-key" + + +def test_get_logging_payload_records_cache_off_without_hashing(monkeypatch): + assert _cache_key_in_spend_log(monkeypatch, None, None) == "Cache OFF" + + +def test_get_logging_payload_still_hashes_when_caching_is_on_and_no_preset_key_exists(monkeypatch): + class _RecordingCache(litellm.Cache): + def __init__(self) -> None: + pass + + def get_cache_key(self, **kwargs) -> str: + return "hashed-from-" + kwargs["model"] + + assert _cache_key_in_spend_log(monkeypatch, _RecordingCache(), None) == "hashed-from-gpt-4o-mini" + + _TRACE_ONLY_STANDARD_LOGGING: Final = cast( StandardLoggingPayload, { @@ -613,11 +655,11 @@ def test_sanitize_request_body_for_spend_logs_payload_circular_reference(): assert sanitized == {"b": {"a": {}}} # Should return empty dict for circular reference -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true( mock_should_store, ): - # When _should_store_prompts_and_responses_in_spend_logs returns True + # When should_store_prompts_and_responses_in_spend_logs returns True mock_should_store.return_value = True # Sample vector store request metadata @@ -631,11 +673,11 @@ def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true( assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == "sensitive information" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false( mock_should_store, ): - # When _should_store_prompts_and_responses_in_spend_logs returns False + # When should_store_prompts_and_responses_in_spend_logs returns False mock_should_store.return_value = False # Sample vector store request metadata @@ -651,7 +693,7 @@ def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false( assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["type"] == "text" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_store): # When input is None mock_should_store.return_value = False @@ -659,7 +701,7 @@ def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_ assert result is None -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store): """ Test that _get_messages_for_spend_logs_payload returns messages @@ -686,7 +728,7 @@ def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store assert parsed[1]["content"] == "What is the weather today?" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_strips_null_bytes(mock_should_store): """Regression for PostgreSQL 22P05: NUL bytes must be stripped from messages.""" mock_should_store.return_value = True @@ -703,7 +745,7 @@ def test_get_messages_for_spend_logs_strips_null_bytes(mock_should_store): assert parsed[0]["content"] == "helloworld" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_realtime_empty_when_disabled(mock_should_store): """ Test that _get_messages_for_spend_logs_payload returns '{}' for realtime calls @@ -721,7 +763,7 @@ def test_get_messages_for_spend_logs_realtime_empty_when_disabled(mock_should_st assert result == "{}" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_non_realtime_returns_empty(mock_should_store): """ Test that _get_messages_for_spend_logs_payload returns '{}' for non-realtime @@ -739,7 +781,7 @@ def test_get_messages_for_spend_logs_non_realtime_returns_empty(mock_should_stor assert result == "{}" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_store): from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB @@ -767,7 +809,7 @@ def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_ assert parsed["data"][0]["other_field"] == "value" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store): """Regression for PostgreSQL 22P05: NUL bytes must be stripped from response.""" mock_should_store.return_value = True @@ -780,7 +822,7 @@ def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store assert json.loads(response_json)["content"] == "answerhere" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_truncates_large_embedding( mock_should_store, ): @@ -835,7 +877,7 @@ def test_truncation_includes_db_safeguard_note(): ) -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_response_truncation_logs_info_message(mock_should_store): """ Test that when response is truncated before DB storage, an info log is emitted @@ -857,7 +899,7 @@ def test_response_truncation_logs_info_message(mock_should_store): assert "response was truncated" in log_msg -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_request_body_truncation_logs_info_message(mock_should_store): """ Test that when request body is truncated before DB storage, an info log is emitted. @@ -1520,7 +1562,7 @@ def test_get_logging_payload_handles_missing_overhead_gracefully(): ) -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_enabled( mock_should_store, ): @@ -1584,7 +1626,7 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin mock_get_secret_bool, ): """ - Test that _should_store_prompts_and_responses_in_spend_logs handles + Test that should_store_prompts_and_responses_in_spend_logs handles case-insensitive string values for store_prompts_in_spend_logs in general_settings. """ # Test case-insensitive string "true" variations @@ -1594,7 +1636,7 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin {"store_prompts_in_spend_logs": true_value}, ): mock_get_secret_bool.return_value = False # Ensure env var is False - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is True, f"Expected True for '{true_value}', got {result}" # Test boolean True @@ -1603,7 +1645,7 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin {"store_prompts_in_spend_logs": True}, ): mock_get_secret_bool.return_value = False - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is True, f"Expected True for boolean True, got {result}" # Test that non-true values fall back to environment variable @@ -1614,22 +1656,22 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin ): # When env var is True, should return True mock_get_secret_bool.return_value = True - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is True, f"Expected True (from env var) for '{false_value}', got {result}" # When env var is False, should return False mock_get_secret_bool.return_value = False - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is False, f"Expected False (from env var) for '{false_value}', got {result}" # Test when general_settings doesn't have the key at all with patch("litellm.proxy.proxy_server.general_settings", {}): mock_get_secret_bool.return_value = True - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is True, "Expected True (from env var) when key missing, got False" mock_get_secret_bool.return_value = False - result = _should_store_prompts_and_responses_in_spend_logs() + result = should_store_prompts_and_responses_in_spend_logs() assert result is False, "Expected False (from env var) when key missing, got True" @@ -1662,7 +1704,7 @@ def test_get_spend_logs_metadata_guardrail_info_fallback_from_metadata(): assert result["guardrail_information"] is None -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_redacts_all_prompt_carrying_fields_when_flag_false( mock_should_store, ): @@ -1698,7 +1740,7 @@ def test_sanitize_guardrail_information_redacts_all_prompt_carrying_fields_when_ assert entry["guardrail_action"] == "NONE" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_redacts_prompt_fields_when_flag_false( mock_should_store, ): @@ -1762,7 +1804,7 @@ def test_sanitize_guardrail_information_redacts_prompt_fields_when_flag_false( } -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_preserves_guardrail_usage_when_flag_false( mock_should_store, ): @@ -1794,7 +1836,7 @@ def test_sanitize_guardrail_information_preserves_guardrail_usage_when_flag_fals assert entry["guardrail_usage"] == {"topicPolicyUnits": 1, "contentPolicyUnits": 1, "wordPolicyUnits": 0} -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_passthrough_when_flag_true( mock_should_store, ): @@ -1817,13 +1859,13 @@ def test_sanitize_guardrail_information_passthrough_when_flag_true( assert result == guardrail_info -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_none_passthrough(mock_should_store): mock_should_store.return_value = False assert _sanitize_guardrail_information_for_spend_logs(None) is None -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_normalizes_bare_dict_input(mock_should_store): """ Regression: xecguard (xecguard.py:246) assigns a bare dict to @@ -1855,7 +1897,7 @@ def test_sanitize_guardrail_information_normalizes_bare_dict_input(mock_should_s assert entry["start_time"] == 1.0 -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_drops_non_dict_items_in_list(mock_should_store): """ A stray non-dict item in the list (e.g. from a buggy caller that @@ -1874,7 +1916,7 @@ def test_sanitize_guardrail_information_drops_non_dict_items_in_list(mock_should assert result == [{"guardrail_name": "x", "guardrail_response": REDACTED_BY_LITELM_STRING}] -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_preserves_absent_prompt_fields(mock_should_store): """ Entries that never carried guardrail_request or guardrail_response must @@ -2290,7 +2332,7 @@ def test_sanitize_request_body_strips_secret_fields(): assert sanitized["messages"] == [{"role": "user", "content": "hi"}] -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_proxy_server_request_payload_excludes_secret_fields(mock_should_store): """ End-to-end test: when the proxy_server_request body contains @@ -2370,7 +2412,7 @@ def test_redact_prompt_leaks_empty_string(): assert _redact_prompt_leaks_in_error_string("") == "" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_redacts_when_not_storing_prompts( mock_should_store, ): @@ -2398,7 +2440,7 @@ def test_sanitize_error_information_redacts_when_not_storing_prompts( assert sanitized["llm_provider"] == "openai" -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_skips_redaction_when_storing_prompts( mock_should_store, ): @@ -2420,7 +2462,7 @@ def test_sanitize_error_information_skips_redaction_when_storing_prompts( assert REDACTED_BY_LITELM_STRING not in sanitized["error_message"] -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_caps_size_regardless_of_prompt_flag( mock_should_store, ): @@ -2451,7 +2493,7 @@ def test_sanitize_error_information_none_passthrough(): assert _sanitize_error_information_for_spend_logs(None) is None -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_reproduces_lit_2992(mock_should_store): # Mirrors the reproduced row body from LIT-2992 — a RateLimitError whose # message embeds 178 pydantic validation errors, each carrying a full @@ -2536,7 +2578,7 @@ def test_redact_prompt_leaks_handles_unterminated_value(): assert REDACTED_BY_LITELM_STRING in redacted -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_redacts_traceback_when_not_storing_prompts( mock_should_store, ): @@ -2568,7 +2610,7 @@ def test_sanitize_error_information_redacts_traceback_when_not_storing_prompts( assert "ValueError: invalid request" in sanitized["traceback"] -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_skips_traceback_redaction_when_storing_prompts( mock_should_store, ): @@ -2676,7 +2718,7 @@ def test_redact_prompt_leaks_combined_quoted_key_and_pydantic_assignment(): assert redacted.count(REDACTED_BY_LITELM_STRING) >= 2 -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_error_information_redacts_pydantic_assignment_form( mock_should_store, ): @@ -3272,7 +3314,7 @@ def test_get_logging_payload_hashes_bearer_prefixed_api_key(): ) -@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_sanitize_guardrail_information_preserves_headroom_compression_token_stats( mock_should_store, ): diff --git a/tests/test_litellm/proxy/test_collector.py b/tests/test_litellm/proxy/test_collector.py new file mode 100644 index 00000000000..ac3e1f5e566 --- /dev/null +++ b/tests/test_litellm/proxy/test_collector.py @@ -0,0 +1,229 @@ +import asyncio +import logging +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Final + +import pytest +import uvloop + +from litellm._logging import verbose_logger, verbose_proxy_logger, verbose_router_logger +from litellm.proxy.collector import ( + SpendEventConsumer, + address_argument, + apply_log_level, + pod_pgbouncer_database_url, +) +from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings +from litellm.proxy.spend_tracking.spend_event_producer import ( + AddressError, + SpendEventProducer, + TcpAddress, + UnixAddress, + open_collector_connection, +) + + +class _Handler: + def __init__(self, fail_on: bytes | None = None) -> None: + self.lines: list[bytes] = [] # mutable-ok: test double records the events the consumer handed over + self._fail_on = fail_on + + async def __call__(self, line: bytes) -> None: + if line == self._fail_on: + raise RuntimeError("pipeline failed") + self.lines.append(line) + + +async def _no_fallback(line: bytes) -> None: + raise AssertionError(f"unexpected fallback for {line!r}") + + +class _Fallback: + def __init__(self) -> None: + self.lines: list[bytes] = [] # mutable-ok: test double records the events run in-process + + async def __call__(self, line: bytes) -> None: + self.lines.append(line) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", ["unix", "tcp"]) +async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path, transport: str): + handler: Final = _Handler(fail_on=b"event-3\n") + consumer: Final = SpendEventConsumer(handler) + server: Final = await consumer.serve( + UnixAddress(path=str(tmp_path / "spend.sock")) if transport == "unix" else TcpAddress("127.0.0.1", 0) + ) + address: Final = ( + UnixAddress(path=str(tmp_path / "spend.sock")) + if transport == "unix" + else TcpAddress("127.0.0.1", server.sockets[0].getsockname()[1]) + ) + producer: Final = SpendEventProducer( + address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=_no_fallback + ) + for i in range(6): + await producer.publish(f"event-{i}\n".encode()) + await producer.close(drain_timeout=5.0) + + server.close() + assert await consumer.drain(timeout=5.0) == 0 + assert handler.lines == [f"event-{i}\n".encode() for i in range(6) if i != 3] + assert (consumer.received, consumer.handled, consumer.failed) == (6, 5, 1) + + +@pytest.mark.asyncio +async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path): + handler: Final = _Handler() + consumer: Final = SpendEventConsumer(handler) + address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + server: Final = await consumer.serve(address) + _, writer = await open_collector_connection(address, timeout=1.0) + writer.write(b"whole\npartial-without-newline") + await writer.drain() + writer.close() + await writer.wait_closed() + await asyncio.sleep(0.05) + + server.close() + assert await consumer.drain(timeout=5.0) == 0 + assert handler.lines == [b"whole\n"] + assert consumer.received == 1 + + +@pytest.mark.asyncio +async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_path: Path): + consumer: Final = SpendEventConsumer(_Handler()) + address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + server: Final = await consumer.serve(address) + _, writer = await open_collector_connection(address, timeout=1.0) + await asyncio.sleep(0.05) + + server.close() + assert await consumer.drain(timeout=0.1) == 1 + writer.close() + await writer.wait_closed() + assert await consumer.drain(timeout=5.0) == 0 + + +@pytest.mark.asyncio +async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(tmp_path: Path): + handler: Final = _Handler() + fallback: Final = _Fallback() + consumer: Final = SpendEventConsumer(handler) + address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + server: Final = await consumer.serve(address) + producer: Final = SpendEventProducer( + address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback + ) + await producer.publish(b"event-1\n") + await asyncio.sleep(0.05) + + server.close() + draining: Final = asyncio.ensure_future(consumer.drain(timeout=5.0)) + await asyncio.sleep(0.05) + await producer.publish(b"event-2\n") + await producer.close(drain_timeout=5.0) + + assert await draining == 0 + assert handler.lines == [b"event-1\n"] + assert fallback.lines == [b"event-2\n"] + assert (producer.stats().sent, producer.stats().fallback) == (1, 1) + + +@pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) +def test_drain_still_hands_over_live_producers_when_another_connection_already_died( + tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] +): + """A transport the loop force-closed under a busy handler must not abort the half-close of the others.""" + + async def scenario() -> tuple[int, list[bytes]]: + release: Final = asyncio.Event() + + async def slow_handler(line: bytes) -> None: + await release.wait() + + consumer: Final = SpendEventConsumer(slow_handler) + address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) + server: Final = await consumer.serve(address) + _, dead = await open_collector_connection(address, timeout=1.0) + dead.write(b"stuck\n") + await dead.drain() + await asyncio.sleep(0.05) + for connection in consumer._open_connections: # pyright: ignore[reportPrivateUsage] # force-close like uvloop does on a socket error + connection.transport.close() + dead.close() + fallback: Final = _Fallback() + producer: Final = SpendEventProducer( + address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback + ) + await producer.publish(b"event-1\n") + await asyncio.sleep(0.05) + + server.close() + draining: Final = asyncio.ensure_future(consumer.drain(timeout=0.5)) + await asyncio.sleep(0.05) + await producer.publish(b"event-2\n") + await producer.close(drain_timeout=5.0) + still_open: Final = await draining + release.set() + await asyncio.sleep(0.05) + return still_open, fallback.lines + + with asyncio.Runner(loop_factory=loop_factory) as runner: + still_open, fallback_lines = runner.run(scenario()) + + assert still_open == 2 + assert fallback_lines == [b"event-2\n"] + + +def test_address_argument(): + assert address_argument((), default="unix:///tmp/x.sock") == "unix:///tmp/x.sock" + assert address_argument(("--address", "tcp://127.0.0.1:4100"), default="unix:///tmp/x.sock") == ( + "tcp://127.0.0.1:4100" + ) + assert isinstance(address_argument(("--listen", "x"), default="unix:///tmp/x.sock"), AddressError) + + +def test_pod_pgbouncer_database_url_points_at_the_proxy_containers_pooler(): + """With pgbouncer on, the sidecar must not open its own upstream connections but share the pod's pooler.""" + upstream: Final = "postgresql://u:p@db.internal:5432/litellm?schema=public" + environ: Final = {"DATABASE_URL": upstream} + assert pod_pgbouncer_database_url(PgBouncerSettings(enabled=False), environ, token_auth=False) is None + assert ( + pod_pgbouncer_database_url(PgBouncerSettings(enabled=True, port=6543), environ, token_auth=False) + == "postgresql://u:p@127.0.0.1:6543/litellm?schema=public&pgbouncer=true" + ) + assert isinstance(pod_pgbouncer_database_url(PgBouncerSettings(enabled=True), {}, token_auth=False), PgBouncerError) + + +def test_pod_pgbouncer_database_url_goes_direct_under_token_auth(): + """The proxy's pgbouncer only knows the token that container minted, so the sidecar must mint its own upstream.""" + iam_upstream: Final = "postgresql://u@db.internal:5432/litellm?schema=public" + assert ( + pod_pgbouncer_database_url(PgBouncerSettings(enabled=True), {"DATABASE_URL": iam_upstream}, token_auth=True) + is None + ) + assert pod_pgbouncer_database_url(PgBouncerSettings(enabled=True), {}, token_auth=True) is None + + +@pytest.fixture +def restore_log_levels() -> Iterator[None]: + loggers: Final = (verbose_logger, verbose_router_logger, verbose_proxy_logger) + levels: Final = tuple(logger.level for logger in loggers) + yield + for logger, level in zip(loggers, levels, strict=True): + logger.setLevel(level) + + +@pytest.mark.usefixtures("restore_log_levels") +@pytest.mark.parametrize( + ("litellm_log", "expected"), + [("DEBUG", logging.DEBUG), ("info", logging.INFO), (None, logging.WARNING), ("loud", logging.WARNING)], +) +def test_apply_log_level_mirrors_the_proxy_env_contract(litellm_log: str | None, expected: int): + verbose_proxy_logger.setLevel(logging.WARNING) + apply_log_level(litellm_log) + assert verbose_proxy_logger.isEnabledFor(expected) + assert not verbose_proxy_logger.isEnabledFor(expected - 10) From 960fc4b114815cd84ac08f2f98d675286e1c9d67 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:16:15 -0700 Subject: [PATCH 387/411] feat(newrelic): export team max and remaining budget gauges to the Metric API (#40542) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/newrelic/newrelic_metrics.py | 39 +++++++- litellm/types/integrations/newrelic.py | 19 +++- .../newrelic/test_newrelic_metrics.py | 92 ++++++++++++++++++- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/newrelic/newrelic_metrics.py b/litellm/integrations/newrelic/newrelic_metrics.py index 25dbfc2bdb2..da952b78d3f 100644 --- a/litellm/integrations/newrelic/newrelic_metrics.py +++ b/litellm/integrations/newrelic/newrelic_metrics.py @@ -5,8 +5,9 @@ NR Reference API: https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-ap `async_log_success_event` / `async_log_failure_event` queue one record per request; at flush the queue is aggregated by (team, model group, model, provider, status) -into count/summary metrics. `interval.ms` is the real window between flushes, -computed at flush time. +into count/summary metrics, plus one max/remaining budget gauge pair per team +taken from the team's latest record. `interval.ms` is the real window between +flushes, computed at flush time. Team-scoped by construction: the ingest key is injected explicitly and there is deliberately no environment-variable fallback, so a team's metrics are never sent @@ -47,11 +48,14 @@ from litellm.types.integrations.newrelic import ( NEWRELIC_METRIC_PROMPT_TOKENS, NEWRELIC_METRIC_REQUEST_DURATION_MS, NEWRELIC_METRIC_REQUESTS, + NEWRELIC_METRIC_TEAM_MAX_BUDGET, + NEWRELIC_METRIC_TEAM_REMAINING_BUDGET, NEWRELIC_METRIC_TOTAL_TOKENS, NEWRELIC_METRICS_MAX_BATCH_SIZE, NEWRELIC_METRICS_MAX_DRAIN_PASSES, NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE, NewRelicCountMetric, + NewRelicGaugeMetric, NewRelicMetric, NewRelicMetricCommon, NewRelicMetricEnvelope, @@ -98,6 +102,8 @@ def _metric_record_from_payload(standard_logging_object: StandardLoggingPayload) completion_tokens=int(standard_logging_object.get("completion_tokens") or 0), total_tokens=int(standard_logging_object.get("total_tokens") or 0), duration_ms=float(standard_logging_object.get("response_time") or 0.0) * 1000.0, + team_max_budget=metadata.get("user_api_key_team_max_budget") if metadata else None, + team_spend=metadata.get("user_api_key_team_spend") if metadata else None, ) @@ -140,6 +146,33 @@ def _bucket_metrics(bucket_records: tuple[NewRelicMetricRecord, ...]) -> tuple[N return (*count_metrics, summary_metric) +def _team_budget_gauges(record: NewRelicMetricRecord) -> tuple[NewRelicMetric, ...]: + team_max_budget: Final = record.team_max_budget + if team_max_budget is None: + return () + attributes: Final[Mapping[str, str]] = { # mutable-ok: JSON leaf; safe_dumps stringifies MappingProxyType + key: value[:NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN] + for key, value in (("team_id", record.team_id), ("team_alias", record.team_alias)) + if value + } + remaining_budget: Final = team_max_budget - (record.team_spend or 0.0) - record.response_cost + return ( + NewRelicGaugeMetric( + name=NEWRELIC_METRIC_TEAM_MAX_BUDGET, type="gauge", value=team_max_budget, attributes=attributes + ), + NewRelicGaugeMetric( + name=NEWRELIC_METRIC_TEAM_REMAINING_BUDGET, type="gauge", value=remaining_budget, attributes=attributes + ), + ) + + +def _team_budget_metrics(records: tuple[NewRelicMetricRecord, ...]) -> tuple[NewRelicMetric, ...]: + latest_by_team: Final[Mapping[str, NewRelicMetricRecord]] = MappingProxyType( + {record.team_id: record for record in records if record.team_id} + ) + return tuple(gauge for record in latest_by_team.values() for gauge in _team_budget_gauges(record)) + + def build_metric_payload( records: tuple[NewRelicMetricRecord, ...], *, @@ -158,7 +191,7 @@ def build_metric_payload( "timestamp": int(window_start * 1000), "interval.ms": interval_ms, } - return (NewRelicMetricEnvelope(common=common, metrics=metrics),) + return (NewRelicMetricEnvelope(common=common, metrics=(*metrics, *_team_budget_metrics(records))),) class NewRelicMetricsLogger(CustomBatchLogger): diff --git a/litellm/types/integrations/newrelic.py b/litellm/types/integrations/newrelic.py index 36e4d02c2a8..b5905ad0b93 100644 --- a/litellm/types/integrations/newrelic.py +++ b/litellm/types/integrations/newrelic.py @@ -27,9 +27,9 @@ NEWRELIC_METRIC_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType( NEWRELIC_DEFAULT_REGION: Final = "us" #: Metric API caps a payload at 2000 data points / 1MB compressed; each queued -#: record expands to at most 6 metrics, so cap the per-flush record count well -#: below that. -NEWRELIC_METRICS_MAX_BATCH_SIZE: Final = 250 +#: record expands to at most 6 bucket metrics plus 2 team budget gauges (8), so cap +#: the per-flush record count well below 2000 / 8. +NEWRELIC_METRICS_MAX_BATCH_SIZE: Final = 200 #: Hard cap on records retained across failed flushes (5xx/network requeue). #: Beyond this the oldest records are dropped. @@ -48,6 +48,8 @@ NEWRELIC_METRIC_PROMPT_TOKENS: Final = "litellm.tokens.prompt" NEWRELIC_METRIC_COMPLETION_TOKENS: Final = "litellm.tokens.completion" NEWRELIC_METRIC_TOTAL_TOKENS: Final = "litellm.tokens.total" NEWRELIC_METRIC_REQUEST_DURATION_MS: Final = "litellm.request.duration_ms" +NEWRELIC_METRIC_TEAM_MAX_BUDGET: Final = "litellm.team.max_budget" +NEWRELIC_METRIC_TEAM_REMAINING_BUDGET: Final = "litellm.team.remaining_budget" class NewRelicSummaryValue(TypedDict): @@ -66,6 +68,13 @@ class NewRelicCountMetric(TypedDict): attributes: ReadOnly[Mapping[str, str]] +class NewRelicGaugeMetric(TypedDict): + name: ReadOnly[str] + type: ReadOnly[Literal["gauge"]] + value: ReadOnly[float] + attributes: ReadOnly[Mapping[str, str]] + + class NewRelicSummaryMetric(TypedDict): name: ReadOnly[str] type: ReadOnly[Literal["summary"]] @@ -73,7 +82,7 @@ class NewRelicSummaryMetric(TypedDict): attributes: ReadOnly[Mapping[str, str]] -NewRelicMetric = NewRelicCountMetric | NewRelicSummaryMetric +NewRelicMetric = NewRelicCountMetric | NewRelicGaugeMetric | NewRelicSummaryMetric #: ``interval.ms`` has a dot in it, so the functional TypedDict form is required. @@ -108,6 +117,8 @@ class NewRelicMetricRecord: completion_tokens: int total_tokens: int duration_ms: float + team_max_budget: float | None = None + team_spend: float | None = None @property def bucket_key(self) -> tuple[str, str, str, str, str, str]: diff --git a/tests/test_litellm/integrations/newrelic/test_newrelic_metrics.py b/tests/test_litellm/integrations/newrelic/test_newrelic_metrics.py index 9c75e0b0a47..a9074e89806 100644 --- a/tests/test_litellm/integrations/newrelic/test_newrelic_metrics.py +++ b/tests/test_litellm/integrations/newrelic/test_newrelic_metrics.py @@ -24,6 +24,8 @@ from litellm.types.integrations.newrelic import ( NEWRELIC_METRIC_PROMPT_TOKENS, NEWRELIC_METRIC_REQUEST_DURATION_MS, NEWRELIC_METRIC_REQUESTS, + NEWRELIC_METRIC_TEAM_MAX_BUDGET, + NEWRELIC_METRIC_TEAM_REMAINING_BUDGET, NEWRELIC_METRIC_TOTAL_TOKENS, NewRelicMetricRecord, ) @@ -40,6 +42,8 @@ def _record( completion_tokens=20, total_tokens=30, duration_ms=100.0, + team_max_budget=None, + team_spend=None, ) -> NewRelicMetricRecord: return NewRelicMetricRecord( team_id=team_id, @@ -53,12 +57,28 @@ def _record( completion_tokens=completion_tokens, total_tokens=total_tokens, duration_ms=duration_ms, + team_max_budget=team_max_budget, + team_spend=team_spend, ) -def _standard_logging_object(team_id="team-a", response_cost=0.25) -> dict: +def _standard_logging_object( + team_id="team-a", response_cost=0.25, team_max_budget: float | None = None, team_spend: float | None = None +) -> dict: + budget_metadata = { + key: value + for key, value in ( + ("user_api_key_team_max_budget", team_max_budget), + ("user_api_key_team_spend", team_spend), + ) + if value is not None + } return { - "metadata": {"user_api_key_team_id": team_id, "user_api_key_team_alias": f"{team_id}-alias"}, + "metadata": { + "user_api_key_team_id": team_id, + "user_api_key_team_alias": f"{team_id}-alias", + **budget_metadata, + }, "model_group": "gpt-4o-group", "model": "gpt-4o", "custom_llm_provider": "openai", @@ -204,6 +224,74 @@ class TestBuildMetricPayload: assert "model_group" not in attributes +class TestTeamBudgetGauges: + def test_latest_record_per_team_drives_one_gauge_pair(self): + records = ( + _record(team_id="team-a", model="gpt-4o", response_cost=0.5, team_max_budget=100.0, team_spend=10.0), + _record(team_id="team-a", model="claude-4", response_cost=2.0, team_max_budget=100.0, team_spend=10.5), + _record(team_id="team-b", response_cost=1.0, team_max_budget=None, team_spend=3.0), + _record(team_id="", team_alias="", response_cost=1.0, team_max_budget=50.0, team_spend=1.0), + ) + payload = build_metric_payload(records, window_start=1_000.0, now=1_005.0) + + max_budget_gauges = _metrics_by_name(payload, NEWRELIC_METRIC_TEAM_MAX_BUDGET) + remaining_gauges = _metrics_by_name(payload, NEWRELIC_METRIC_TEAM_REMAINING_BUDGET) + assert [(m["type"], m["value"], m["attributes"]) for m in max_budget_gauges] == [ + ("gauge", 100.0, {"team_id": "team-a", "team_alias": "team-a-alias"}) + ] + assert [(m["type"], m["attributes"]) for m in remaining_gauges] == [ + ("gauge", {"team_id": "team-a", "team_alias": "team-a-alias"}) + ] + assert remaining_gauges[0]["value"] == pytest.approx(100.0 - 10.5 - 2.0) + assert len(_metrics_by_name(payload, NEWRELIC_METRIC_COST_USD)) == 4 + + def test_missing_team_spend_counts_only_this_request(self): + payload = build_metric_payload( + (_record(response_cost=0.25, team_max_budget=10.0, team_spend=None),), window_start=1_000.0, now=1_005.0 + ) + + assert _metrics_by_name(payload, NEWRELIC_METRIC_TEAM_REMAINING_BUDGET)[0]["value"] == pytest.approx(9.75) + + @pytest.mark.asyncio + async def test_budget_gauges_reach_the_metric_api_from_standard_logging_metadata(self): + logger = _make_logger() + logger.async_client.post = AsyncMock(return_value=_response(202)) + slo = _standard_logging_object(response_cost=0.25, team_max_budget=20.0, team_spend=4.5) + + await logger.async_log_success_event( + kwargs={"standard_logging_object": slo}, response_obj={}, start_time=None, end_time=None + ) + await logger.flush_queue() + + body = json.loads(gzip.decompress(logger.async_client.post.await_args.kwargs["data"]).decode("utf-8")) + by_name = {m["name"]: m for m in body[0]["metrics"]} + assert by_name[NEWRELIC_METRIC_TEAM_MAX_BUDGET] == { + "name": NEWRELIC_METRIC_TEAM_MAX_BUDGET, + "type": "gauge", + "value": 20.0, + "attributes": {"team_id": "team-a", "team_alias": "team-a-alias"}, + } + assert by_name[NEWRELIC_METRIC_TEAM_REMAINING_BUDGET]["type"] == "gauge" + assert by_name[NEWRELIC_METRIC_TEAM_REMAINING_BUDGET]["value"] == pytest.approx(15.25) + assert by_name[NEWRELIC_METRIC_COST_USD]["value"] == 0.25 + + @pytest.mark.asyncio + async def test_no_budget_metadata_sends_no_gauges(self): + logger = _make_logger() + logger.async_client.post = AsyncMock(return_value=_response(202)) + + await logger.async_log_success_event( + kwargs={"standard_logging_object": _standard_logging_object()}, + response_obj={}, + start_time=None, + end_time=None, + ) + await logger.flush_queue() + + body = json.loads(gzip.decompress(logger.async_client.post.await_args.kwargs["data"]).decode("utf-8")) + assert {m["type"] for m in body[0]["metrics"]} == {"count", "summary"} + + class TestQueueAndFlush: @pytest.mark.asyncio async def test_log_event_queues_record_from_standard_logging_object(self): From 1f7c4d6784411efa2c9622aa6ea7dcf3e0320e89 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 10 Sep 2026 17:18:56 -0700 Subject: [PATCH 388/411] test: fix stale completion response fixtures --- tests/llm_translation/test_azure_o_series.py | 8 ++++++++ tests/llm_translation/test_azure_openai.py | 5 +++++ tests/llm_translation/test_openai.py | 6 ++++++ tests/local_testing/test_completion.py | 9 +++++++-- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/llm_translation/test_azure_o_series.py b/tests/llm_translation/test_azure_o_series.py index 1a2d672af71..ce7e614cbe2 100644 --- a/tests/llm_translation/test_azure_o_series.py +++ b/tests/llm_translation/test_azure_o_series.py @@ -159,15 +159,23 @@ def test_azure_o_series_routing(): def test_openai_o_series_max_retries_0(mock_get_openai_client): import litellm + mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.headers = {} + mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.parse.return_value = ( + ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}]) + ) litellm.set_verbose = True response = litellm.completion( model="azure/o1-preview", messages=[{"role": "user", "content": "hi"}], max_retries=0, + api_key="fake-key", + api_base="https://fake-azure.openai.azure.com", + api_version="2024-10-21", ) mock_get_openai_client.assert_called_once() assert mock_get_openai_client.call_args.kwargs["max_retries"] == 0 + assert response.choices[0].message.content == "Hello" @pytest.mark.asyncio diff --git a/tests/llm_translation/test_azure_openai.py b/tests/llm_translation/test_azure_openai.py index 0fa72b45ed8..e6528e77749 100644 --- a/tests/llm_translation/test_azure_openai.py +++ b/tests/llm_translation/test_azure_openai.py @@ -335,6 +335,10 @@ def test_azure_gpt_4o_with_tool_call_and_response_format(api_version): ] with patch.object(client.chat.completions.with_raw_response, "create") as mock_post: + mock_post.return_value.headers = {} + mock_post.return_value.parse.return_value = litellm.ModelResponse( + choices=[{"message": {"role": "assistant", "content": InvestigationOutput().model_dump_json()}}] + ) response = litellm.completion( model="azure/gpt-4.1-mini", messages=[ @@ -362,6 +366,7 @@ def test_azure_gpt_4o_with_tool_call_and_response_format(api_version): assert "response_format" in mock_post.call_args.kwargs else: assert "response_format" not in mock_post.call_args.kwargs + assert response.choices[0].message.content == InvestigationOutput().model_dump_json() def test_map_openai_params(): diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 2b9abdec5d0..af4ba85d58e 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -292,15 +292,21 @@ class TestOpenAIChatCompletion(BaseLLMChatTest): def test_openai_max_retries_0(mock_get_openai_client): import litellm + mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.headers = {} + mock_get_openai_client.return_value.chat.completions.with_raw_response.create.return_value.parse.return_value = ( + ModelResponse(choices=[{"message": {"role": "assistant", "content": "Hello"}}]) + ) litellm.set_verbose = True response = litellm.completion( model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}], max_retries=0, + api_key="fake-key", ) mock_get_openai_client.assert_called_once() assert mock_get_openai_client.call_args.kwargs["max_retries"] == 0 + assert response.choices[0].message.content == "Hello" @patch("litellm.main.openai_chat_completions._get_openai_client") diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index f8f23ea015a..43ed57f63af 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -3999,10 +3999,14 @@ def test_completion_novita_ai(): openai_client = OpenAI(api_key="fake-key") with patch.object( - openai_client.chat.completions, "create", new=MagicMock() + openai_client.chat.completions.with_raw_response, "create" ) as mock_call: + mock_call.return_value.headers = {} + mock_call.return_value.parse.return_value = litellm.ModelResponse( + choices=[{"message": {"role": "assistant", "content": "Hello"}}] + ) try: - completion( + response = completion( model="novita/meta-llama/llama-3.3-70b-instruct", messages=messages, client=openai_client, @@ -4010,6 +4014,7 @@ def test_completion_novita_ai(): ) mock_call.assert_called_once() + assert response.choices[0].message.content == "Hello" # Verify model is passed correctly assert ( From c7a41c35d5cdf2fc2d270c6fee6c793729eeee06 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:48:25 +0000 Subject: [PATCH 389/411] perf(mock): emit admission-time usage chunk on streaming mock_response (#40637) * perf(mock): emit admission-time usage chunk on streaming mock_response Streaming mock_response chunks carried no usage, so the chunk builder re-tokenized the whole prompt in Python after the stream ended even when budget reservation had already counted it at admission. The mock streaming generators now yield a final usage-only chunk carrying the admission prompt count (same completion count as the non-streaming path). Without an admission count the old tokenizer fallback stays. * fix(mock): type the mock stream generators and keep the usage chunk on the content stream id Review follow-up: the usage-only chunk was built with a fresh id, so CustomStreamWrapper switched response_id for the finish-reason and usage chunks. It now copies the content stream id. The generators also get full parameter and return annotations. --------- Co-authored-by: yassin --- litellm/main.py | 4 +- litellm/utils.py | 35 +++- .../test_streaming_chunk_builder_utils.py | 49 ++++++ tests/test_litellm/test_main.py | 158 ++++++++++++++++++ tests/test_litellm/test_utils.py | 90 ++++++++++ 5 files changed, 329 insertions(+), 7 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 1a4beb787bc..10fb32828f1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -939,7 +939,7 @@ def mock_completion( if kwargs.get("acompletion", False) is True: return CustomStreamWrapper( completion_stream=async_mock_completion_streaming_obj( - model_response, mock_response=mock_response, model=model, n=n + model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens ), model=model, custom_llm_provider="openai", @@ -947,7 +947,7 @@ def mock_completion( ) return CustomStreamWrapper( completion_stream=mock_completion_streaming_obj( - model_response, mock_response=mock_response, model=model, n=n + model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens ), model=model, custom_llm_provider="openai", diff --git a/litellm/utils.py b/litellm/utils.py index aced4c9b312..b93938a480e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -66,6 +66,7 @@ from litellm.constants import ( DEFAULT_EMBEDDING_PARAM_VALUES, DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT, + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_TRIM_RATIO, FUNCTION_DEFINITION_TOKEN_COUNT, @@ -278,7 +279,7 @@ except (ImportError, AttributeError, TypeError): # Convert to str (if necessary) claude_json_str = json.dumps(json_data) import importlib.metadata -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args from litellm import utils as litellm_utils @@ -7001,7 +7002,26 @@ class TextCompletionStreamWrapper: raise StopAsyncIteration -def mock_completion_streaming_obj(model_response, mock_response, model, n: int | None = None): +def mock_stream_usage_chunk(model_response: ModelResponseStream, model: str, prompt_tokens: int) -> ModelResponseStream: + return ModelResponseStream( + id=model_response.id, + choices=[], # mutable-ok: ModelResponseStream only treats a list as explicit choices, a tuple gets a default choice + model=model, + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + total_tokens=prompt_tokens + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + ), + ) + + +def mock_completion_streaming_obj( + model_response: ModelResponseStream, + mock_response: str | MockException | ModelResponseStream, + model: str, + n: int | None = None, + prompt_tokens: int | None = None, +) -> Iterator[ModelResponseStream]: if isinstance(mock_response, litellm.MockException): raise mock_response if isinstance(mock_response, ModelResponseStream): @@ -7021,14 +7041,17 @@ def mock_completion_streaming_obj(model_response, mock_response, model, n: int | _all_choices.append(_streaming_choice) model_response.choices = _all_choices yield model_response + if prompt_tokens is not None: + yield mock_stream_usage_chunk(model_response, model=model, prompt_tokens=prompt_tokens) async def async_mock_completion_streaming_obj( - model_response, + model_response: ModelResponseStream, mock_response: str | MockException | ModelResponseStream, - model, + model: str, n: int | None = None, -): + prompt_tokens: int | None = None, +) -> AsyncIterator[ModelResponseStream]: if isinstance(mock_response, litellm.MockException): raise mock_response if isinstance(mock_response, ModelResponseStream): @@ -7048,6 +7071,8 @@ async def async_mock_completion_streaming_obj( _all_choices.append(_streaming_choice) model_response.choices = _all_choices yield model_response + if prompt_tokens is not None: + yield mock_stream_usage_chunk(model_response, model=model, prompt_tokens=prompt_tokens) ########## Reading Config File ############################ diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 626b8a63b20..efe4209c1c9 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1554,3 +1554,52 @@ def test_stream_chunk_builder_reads_role_from_first_frame_with_choices() -> None assert response is not None assert response.choices[0].message.role == "user" assert response.choices[0].message.content == "Hi" + + +def _fail_prompt_token_count() -> int: + raise AssertionError("prompt tokens must come from the usage chunk, not the tokenizer") + + +def test_calculate_usage_reads_prompt_tokens_from_mock_stream_usage_chunk_without_tokenizer_fallback() -> None: + from litellm.utils import mock_completion_streaming_obj + + chunks: Final = list( + mock_completion_streaming_obj( + ModelResponseStream(model="gpt-5.4-mini"), + mock_response="ok", + model="gpt-5.4-mini", + prompt_tokens=51234, + ) + ) + assert chunks[-1].choices == [] + + usage: Final = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, + model="gpt-5.4-mini", + completion_output="ok", + count_prompt_tokens=_fail_prompt_token_count, + ) + + assert usage.prompt_tokens == 51234 + assert usage.completion_tokens == chunks[-1].usage.completion_tokens + assert usage.total_tokens == 51234 + usage.completion_tokens + + +def test_calculate_usage_falls_back_to_prompt_counter_when_mock_stream_has_no_admission_count() -> None: + from litellm.utils import mock_completion_streaming_obj + + chunks: Final = list( + mock_completion_streaming_obj( + ModelResponseStream(model="gpt-5.4-mini"), mock_response="ok", model="gpt-5.4-mini" + ) + ) + assert all(chunk.choices for chunk in chunks) + + usage: Final = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, + model="gpt-5.4-mini", + completion_output="ok", + count_prompt_tokens=lambda: 77, + ) + + assert usage.prompt_tokens == 77 diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index f8841af9750..a36ca229981 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2421,6 +2421,164 @@ def test_mock_completion_usage_falls_back_to_default_without_admission_count(): assert response.usage.prompt_tokens == litellm_main.DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT +_ADMISSION_INPUT_TOKENS: Final = 51234 +_ADMISSION_METADATA: Final = { + "user_api_key_budget_reservation": {"reserved_cost": 1.0, "input_tokens": _ADMISSION_INPUT_TOKENS} +} +_MOCK_STREAM_MESSAGES: Final = [{"role": "user", "content": "hello " * 200}] +_STREAM_CHUNK_BUILDER_TOKEN_COUNTER: Final = "litellm.litellm_core_utils.streaming_chunk_builder_utils.token_counter" + + +def _prompt_token_counter_calls(token_counter: MagicMock) -> list[object]: + return [call for call in token_counter.call_args_list if call.kwargs.get("messages") is not None] + + +def _client_usage_chunks(chunks: list[ModelResponseStream]) -> list[Usage]: + return [chunk.usage for chunk in chunks if getattr(chunk, "usage", None) is not None] + + +@pytest.mark.parametrize("n", (None, 2)) +def test_mock_completion_stream_usage_reports_admission_input_tokens_without_tokenizer_fallback(n: int | None): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + n=n, + stream_options={"include_usage": True}, + metadata=_ADMISSION_METADATA, + ) + ) + + usage_chunks = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert usage_chunks[0].prompt_tokens == _ADMISSION_INPUT_TOKENS + assert usage_chunks[0].completion_tokens == litellm_main.DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT + assert usage_chunks[0].total_tokens == _ADMISSION_INPUT_TOKENS + usage_chunks[0].completion_tokens + assert _prompt_token_counter_calls(token_counter) == [] + assert all(chunk.choices for chunk in chunks[:-1]) + assert {chunk.id for chunk in chunks} == {chunks[0].id} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("n", (None, 2)) +async def test_mock_acompletion_stream_usage_reports_admission_input_tokens_without_tokenizer_fallback( + n: int | None, +): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + response = await litellm.acompletion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + n=n, + stream_options={"include_usage": True}, + litellm_metadata=_ADMISSION_METADATA, + ) + chunks = [chunk async for chunk in response] + + usage_chunks = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert usage_chunks[0].prompt_tokens == _ADMISSION_INPUT_TOKENS + assert usage_chunks[0].total_tokens == _ADMISSION_INPUT_TOKENS + usage_chunks[0].completion_tokens + assert _prompt_token_counter_calls(token_counter) == [] + assert all(chunk.choices for chunk in chunks[:-1]) + assert {chunk.id for chunk in chunks} == {chunks[0].id} + + +def test_mock_completion_stream_without_include_usage_hides_usage_chunk_but_logs_admission_count(): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + metadata=_ADMISSION_METADATA, + ) + ) + + assert _client_usage_chunks(chunks) == [] + assert all(len(chunk.choices) == 1 for chunk in chunks) + assert chunks[-1]._hidden_params["usage"].prompt_tokens == _ADMISSION_INPUT_TOKENS + assert _prompt_token_counter_calls(token_counter) == [] + + +def test_mock_completion_stream_without_admission_count_falls_back_to_tokenizer(): + expected_prompt_tokens: Final = litellm.token_counter(model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES) + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + metadata={"user_api_key_budget_reservation": {"reserved_cost": 1.0}}, + ) + ) + + usage_chunks = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert usage_chunks[0].prompt_tokens == expected_prompt_tokens + assert usage_chunks[0].total_tokens == expected_prompt_tokens + usage_chunks[0].completion_tokens + assert len(_prompt_token_counter_calls(token_counter)) >= 1 + + +@pytest.mark.asyncio +async def test_mock_acompletion_stream_without_admission_count_falls_back_to_tokenizer(): + expected_prompt_tokens: Final = litellm.token_counter(model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES) + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + response = await litellm.acompletion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + ) + chunks = [chunk async for chunk in response] + + usage_chunks = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert usage_chunks[0].prompt_tokens == expected_prompt_tokens + assert len(_prompt_token_counter_calls(token_counter)) >= 1 + + +def test_mock_completion_stream_and_non_stream_report_the_same_admission_usage(): + non_stream = litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + metadata=_ADMISSION_METADATA, + ) + chunks = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + metadata=_ADMISSION_METADATA, + ) + ) + + stream_usage: Final = _client_usage_chunks(chunks)[0] + assert (non_stream.usage.prompt_tokens, non_stream.usage.completion_tokens, non_stream.usage.total_tokens) == ( + stream_usage.prompt_tokens, + stream_usage.completion_tokens, + stream_usage.total_tokens, + ) + + def test_mock_completion_stream_with_model_response(): """Test that mock_completion correctly handles stream=True with a ModelResponse as mock_response.""" from litellm import completion diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 8b186be43e5..2bdde43ff2d 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -15,6 +15,7 @@ from jsonschema import validate import litellm from litellm._internal_context import is_internal_call +from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT from litellm._logging import ( CorrelationContextFilter, JsonFormatter, @@ -6208,3 +6209,92 @@ def test_load_credentials_from_list_fills_kwargs_from_the_loaded_credential_with "api_key": "sk-from-db", } assert _credential_warnings(caplog) == [] + + +_MOCK_STREAM_ID: Final = "chatcmpl-mock-stream" +_ChunkSnapshot = tuple[str, tuple[str | None, ...], Usage | None] + + +def _snapshot(chunk: ModelResponseStream) -> _ChunkSnapshot: + return chunk.id, tuple(choice.delta.content for choice in chunk.choices), getattr(chunk, "usage", None) + + +def _mock_stream_snapshots(mock_response: object, prompt_tokens: int | None) -> list[_ChunkSnapshot]: + from litellm.utils import mock_completion_streaming_obj + + return [ + _snapshot(chunk) + for chunk in mock_completion_streaming_obj( + ModelResponseStream(id=_MOCK_STREAM_ID, model="gpt-5.4-mini"), + mock_response=mock_response, + model="gpt-5.4-mini", + prompt_tokens=prompt_tokens, + ) + ] + + +async def _async_mock_stream_snapshots(mock_response: object, prompt_tokens: int | None) -> list[_ChunkSnapshot]: + from litellm.utils import async_mock_completion_streaming_obj + + return [ + _snapshot(chunk) + async for chunk in async_mock_completion_streaming_obj( + ModelResponseStream(id=_MOCK_STREAM_ID, model="gpt-5.4-mini"), + mock_response=mock_response, + model="gpt-5.4-mini", + prompt_tokens=prompt_tokens, + ) + ] + + +_CONTENT_SNAPSHOTS: Final = [(_MOCK_STREAM_ID, (content,), None) for content in ("hel", "lo ", "wor", "ld")] + + +def _assert_trailing_usage_chunk(snapshots: list[_ChunkSnapshot], prompt_tokens: int) -> None: + assert snapshots[:-1] == _CONTENT_SNAPSHOTS + chunk_id, choices, usage = snapshots[-1] + assert chunk_id == _MOCK_STREAM_ID + assert choices == () + assert usage is not None + assert usage.prompt_tokens == prompt_tokens + assert usage.completion_tokens == DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT + assert usage.total_tokens == prompt_tokens + usage.completion_tokens + + +@pytest.mark.parametrize("prompt_tokens", (51234, 0)) +def test_mock_completion_streaming_obj_emits_usage_chunk_with_admission_prompt_tokens(prompt_tokens: int) -> None: + _assert_trailing_usage_chunk(_mock_stream_snapshots("hello world", prompt_tokens), prompt_tokens) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prompt_tokens", (51234, 0)) +async def test_async_mock_completion_streaming_obj_emits_usage_chunk_with_admission_prompt_tokens( + prompt_tokens: int, +) -> None: + _assert_trailing_usage_chunk(await _async_mock_stream_snapshots("hello world", prompt_tokens), prompt_tokens) + + +def test_mock_completion_streaming_obj_emits_no_usage_chunk_without_admission_prompt_tokens() -> None: + assert _mock_stream_snapshots("hello world", None) == _CONTENT_SNAPSHOTS + + +@pytest.mark.asyncio +async def test_async_mock_completion_streaming_obj_emits_no_usage_chunk_without_admission_prompt_tokens() -> None: + assert await _async_mock_stream_snapshots("hello world", None) == _CONTENT_SNAPSHOTS + + +def test_mock_completion_streaming_obj_passes_prebuilt_stream_chunk_through_without_usage_chunk() -> None: + prebuilt: Final = ModelResponseStream( + model="gpt-5.4-mini", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="prebuilt"))] + ) + + assert _mock_stream_snapshots(prebuilt, 51234) == [(prebuilt.id, ("prebuilt",), None)] + + +@pytest.mark.asyncio +async def test_async_mock_completion_streaming_obj_raises_mock_exception_before_usage_chunk() -> None: + mock_exception: Final = litellm.MockException( + status_code=500, message="boom", llm_provider="openai", model="gpt-5.4-mini" + ) + with pytest.raises(litellm.MockException): + await _async_mock_stream_snapshots(mock_exception, 51234) From 985ac6b6a514d8abec939b897720326a8404eb69 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:48:46 -0700 Subject: [PATCH 390/411] fix(rate_limiter): skip non-Latin-1 x-litellm-priority header on /v1/messages (#40636) A team or key priority that is not Latin-1 encodable (for example CJK text) was attached as a response header by the dynamic rate limiter v3 post-call hook, and Starlette then raised UnicodeEncodeError while writing headers, turning a successful /v1/messages call into HTTP 500. The header is now omitted for such values while x-litellm-rate-limiter-version and the v3 rate limit headers are still attached. Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 12 ++++++++- .../hooks/test_dynamic_rate_limiter_v3.py | 27 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index a074f02f4e8..0339cf4dfea 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -59,6 +59,10 @@ def _get_priority_settings() -> "PriorityReservationSettings": return settings +def _is_latin1_encodable(value: object) -> bool: + return all(ord(char) < 256 for char in str(value)) + + class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): """ Saturation-aware priority-based rate limiter using v3 infrastructure. @@ -666,7 +670,13 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if response_has_hidden_params(response): priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) additional_headers: Final = ensure_response_additional_headers(response) - additional_headers["x-litellm-priority"] = priority or "default" + priority_header: Final = priority or "default" + if _is_latin1_encodable(priority_header): + additional_headers["x-litellm-priority"] = priority_header + else: + verbose_proxy_logger.debug( + "Skipping x-litellm-priority header: priority %r is not Latin-1 encodable", priority + ) additional_headers["x-litellm-rate-limiter-version"] = "v3" return response diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 0cd6b4ede9c..527449bbc48 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1918,3 +1918,30 @@ async def test_post_call_success_hook_leaves_raw_provider_dict_untouched(): ) assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_metadata", "expected_priority_header"), + [ + ({"priority": "优先"}, None), + ({"priority": "high"}, "high"), + ({}, "default"), + ], +) +async def test_post_call_success_hook_priority_header_is_always_http_encodable(team_metadata, expected_priority_header): + from starlette.responses import Response + + handler = DynamicRateLimitHandler(internal_usage_cache=DualCache()) + response = {"id": "msg_123", "type": "message", "role": "assistant", "content": [], "_hidden_params": {}} + + await handler.async_post_call_success_hook( + data={"model": "anthropic-haiku"}, + user_api_key_dict=UserAPIKeyAuth(team_id="team-1", team_metadata=team_metadata), + response=response, + ) + + additional_headers = response["_hidden_params"]["additional_headers"] + http_response = Response(headers={key: str(value) for key, value in additional_headers.items()}) + assert http_response.headers.get("x-litellm-priority") == expected_priority_header + assert http_response.headers["x-litellm-rate-limiter-version"] == "v3" From e9c388d79917cd4018a568181f31969ab223cf4e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:49:20 -0700 Subject: [PATCH 391/411] fix(caching): keep an open Redis circuit breaker quiet on the sync read and spend counter paths The sync get path was unguarded, logged with a stray format argument, and never fed the breaker. The sync batch read swallowed the breaker's refusal as an ERROR plus a service failure event per call, so DualCache dropped its in-memory hits and left batch reservations behind. record_success closed an OPEN breaker on stale in-flight successes, skipping the recovery timeout and the half-open probe. The spend counter pipeline re-raised the refusal into the cost callback, which logged an ERROR and fired the failed-tracking alert per request. --- litellm/caching/dual_cache.py | 12 ++- litellm/caching/redis_cache.py | 9 +- litellm/proxy/management_endpoints/ui_sso.py | 18 ++-- litellm/proxy/proxy_server.py | 5 +- tests/test_litellm/caching/test_dual_cache.py | 27 ++++++ .../test_litellm/caching/test_redis_cache.py | 83 +++++++++++++++++-- .../proxy/management_endpoints/test_ui_sso.py | 22 +++++ .../proxy/proxy_server/test_spend_counters.py | 48 +++++++++++ 8 files changed, 205 insertions(+), 19 deletions(-) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index baabfad6852..be761e1258b 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -23,7 +23,7 @@ from litellm.constants import DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE from .base_cache import BaseCache from .in_memory_cache import InMemoryCache -from .redis_cache import RedisCache, log_redis_failure +from .redis_cache import RedisCache, RedisCircuitBreakerOpenError, log_redis_failure if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -206,9 +206,12 @@ class DualCache(BaseCache): redis_result: Final = self.redis_cache.batch_get_cache( key_list=sublist_keys, parent_otel_span=parent_otel_span ) - except Exception: + except Exception as e: # Do not throttle subsequent callers if the Redis read fails. self._rollback_redis_batch_key_reservations(previous_access_times) + if isinstance(e, RedisCircuitBreakerOpenError): + verbose_logger.debug("LiteLLM Cache: batch_get_cache served from memory only: %s", e) + return result raise if self.in_memory_cache is not None: @@ -325,9 +328,12 @@ class DualCache(BaseCache): redis_result: Final = await self.redis_cache.async_batch_get_cache( sublist_keys, parent_otel_span=parent_otel_span ) - except Exception: + except Exception as e: # Do not throttle subsequent callers if the Redis read fails. self._rollback_redis_batch_key_reservations(previous_access_times) + if isinstance(e, RedisCircuitBreakerOpenError): + verbose_logger.debug("LiteLLM Cache: async_batch_get_cache served from memory only: %s", e) + return result raise # Short-circuit if redis_result is None or contains only None values diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 6b93529e456..5b1a9739eb8 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -250,7 +250,7 @@ class RedisCircuitBreaker: self._set_state(self.OPEN) def record_success(self) -> None: - if not self.enabled: + if not self.enabled or self._state == self.OPEN: return if self._state == self.HALF_OPEN: verbose_logger.info("Redis circuit breaker CLOSED — Redis recovered") @@ -1337,6 +1337,7 @@ class RedisCache(BaseCache): except Exception: return ast.literal_eval(decoded) + @_redis_circuit_breaker_guard_sync def get_cache(self, key, parent_otel_span: Span | None = None, **kwargs): try: key = self.check_and_fix_namespace(key=key) @@ -1356,8 +1357,8 @@ class RedisCache(BaseCache): print_verbose(f"Got Redis Cache: key: {key}, cached_response {cached_response}") return self._get_cache_logic(cached_response=cached_response) except Exception as e: - # NON blocking - notify users Redis is throwing an exception - verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: ", e) + verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: %s", e) + _record_swallowed_redis_failure(self._circuit_breaker, e) def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]: """ @@ -1394,9 +1395,9 @@ class RedisCache(BaseCache): key_value_dict = {} _key_list: Final = [key for key in key_list if key is not None] start_time: Final = time.time() + swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") try: - swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list] results: Final = self._run_redis_mget_operation(keys=_keys) _exit_circuit_breaker(self._circuit_breaker, swallowed_before) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index c60888e298f..1ba90725eff 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -47,6 +47,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.caching.dual_cache import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import ( CLI_SSO_CLAIM_MAP, CLI_SSO_CLAIM_MAX_SCALAR_LENGTH, @@ -336,6 +337,16 @@ def _check_cli_sso_start_rate_limit( ) +def _read_cli_sso_flow(cache: DualCache, cache_key: str) -> object: + redis_cache: Final = cache.redis_cache + if redis_cache is None: + return cache.get_cache(key=cache_key) + try: + return redis_cache.get_cache(key=cache_key) + except RedisCircuitBreakerOpenError: + return None + + def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict: if isinstance(login_id, str) and login_id.startswith("sk-"): raise HTTPException( @@ -348,12 +359,7 @@ def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict: if not _is_valid_cli_sso_login_id(login_id): raise HTTPException(status_code=400, detail="Invalid CLI login session id") - cache_key: Final = _get_cli_sso_flow_cache_key(cast(str, login_id)) - redis_cache: Final = cache.redis_cache - if redis_cache is not None: - flow = redis_cache.get_cache(key=cache_key) - else: - flow = cache.get_cache(key=cache_key) + flow = _read_cli_sso_flow(cache, _get_cli_sso_flow_cache_key(cast(str, login_id))) if isinstance(flow, str): try: flow = _as_object(json.loads(flow)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0f24acb8bb4..94e74b20297 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -251,6 +251,7 @@ import litellm._redis from litellm import Router from litellm._logging import _redact_string, verbose_proxy_logger, verbose_router_logger from litellm.caching.caching import DualCache, RedisCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.constants import ( _REALTIME_BODY_CACHE_SIZE, @@ -3412,8 +3413,10 @@ async def _apply_spend_counter_increments(pending: Sequence[_PendingSpendIncreme ] try: results: Final = await redis_cache.async_increment_pipeline(increment_list=increment_list) - except Exception: + except Exception as e: await asyncio.gather(*(_invalidate_spend_counter(counter_key=item.counter_key) for item in pending)) + if isinstance(e, RedisCircuitBreakerOpenError): + return raise for item, current_value in zip(pending, results or ()): spend_counter_cache.in_memory_cache.set_cache(key=item.counter_key, value=current_value) diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index eae8bbfdaff..4c9068722b8 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -677,3 +677,30 @@ async def test_a_real_redis_failure_still_logs_an_error(caplog): errors = [record for record in caplog.records if record.levelno == logging.ERROR] assert [record.getMessage() for record in errors] == ["LiteLLM Cache: exception in async_get_cache: redis is down"] assert errors[0].exc_info is not None + + +def _dual_cache_with_open_breaker_and_a_memory_hit() -> DualCache: + in_memory = InMemoryCache() + in_memory.set_cache("k1", "v1") + return DualCache(in_memory_cache=in_memory, redis_cache=_OpenBreakerRedis(), default_redis_batch_cache_expiry=10) # pyright: ignore[reportArgumentType] # duck-typed Redis double + + +def test_open_breaker_keeps_sync_batch_read_memory_hits_and_releases_reservations(): + """A refused Redis batch read must still answer with the in-memory hits and hold no reservation. + + The refusal was logged and turned into a bare None, so a caller lost its in-memory hits + for as long as the breaker stayed open, and the reserved keys stayed throttled until + the batch expiry passed even though nothing was ever read for them. + """ + cache = _dual_cache_with_open_breaker_and_a_memory_hit() + + assert list(cache.batch_get_cache(["k1", "k2"])) == ["v1", None] + assert "k2" not in cache.last_redis_batch_access_time + + +@pytest.mark.asyncio +async def test_open_breaker_keeps_async_batch_read_memory_hits_and_releases_reservations(): + cache = _dual_cache_with_open_breaker_and_a_memory_hit() + + assert list(await cache.async_batch_get_cache(["k1", "k2"])) == ["v1", None] + assert "k2" not in cache.last_redis_batch_access_time diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index bcaa58c9c40..c6f0caa1dca 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1,11 +1,12 @@ import asyncio +import time from collections.abc import Iterator from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm._service_logger import ServiceLogging -from litellm.caching.redis_cache import RedisCache +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError @pytest.fixture @@ -515,14 +516,46 @@ async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_met await call_method(cache) -def test_circuit_breaker_open_keeps_sync_batch_get_cache_as_a_miss(sync_batch_redis_cache): - """An open breaker must preserve the sync batch read's dictionary fallback.""" +def test_circuit_breaker_open_makes_sync_batch_get_cache_fast_fail(sync_batch_redis_cache, caplog): + """Once the breaker is open the sync batch read refuses with the typed error instead of a miss. + + Swallowing the refusal into `{}` made every sync batch read on an open breaker emit an ERROR + log and a service failure event per call, and the DualCache caller could not tell the + refusal from a dead Redis, so it dropped its in-memory hits too. + """ from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {} - assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {} + caplog.clear() + with caplog.at_level("INFO"): + with pytest.raises(RedisCircuitBreakerOpenError): + sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) + sync_batch_redis_cache.redis_client.mget.assert_called() + assert caplog.records == [] + + +def test_sync_get_cache_failure_feeds_the_breaker_and_logs_a_well_formed_record(sync_batch_redis_cache, caplog): + """The sync get path swallowed its Redis error without recording it, and its log call was malformed. + + `verbose_logger.error("...: ", e)` passes the exception as a format argument to a message + with no placeholder, so the record carried no error text. Nothing fed the breaker either, + so a dead Redis read through this path never opened it. + """ + from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + + sync_batch_redis_cache.redis_client.get.side_effect = OSError("redis unavailable") + + with caplog.at_level("ERROR"): + for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD): + assert sync_batch_redis_cache.get_cache("lit7468") is None + + assert all("redis unavailable" in record.getMessage() for record in caplog.records) + assert len(caplog.records) == REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD + assert sync_batch_redis_cache._circuit_breaker.is_open() is True + with pytest.raises(RedisCircuitBreakerOpenError): + sync_batch_redis_cache.get_cache("lit7468") def test_batch_get_counts_raises_where_batch_get_cache_reports_a_miss(sync_batch_redis_cache): @@ -661,7 +694,8 @@ def test_sync_batch_get_cache_survives_a_service_callback_that_raises( with ThreadPoolExecutor(max_workers=1) as pool: assert pool.submit(cache.batch_get_cache, key_list=["lit6729"]).result() == {} - assert cache.batch_get_cache(key_list=["lit6729"]) == {} + with pytest.raises(RedisCircuitBreakerOpenError): + cache.batch_get_cache(key_list=["lit6729"]) def test_call_stack_info_skips_breaker_guard_frames(): @@ -1010,6 +1044,8 @@ async def test_breaker_metrics_track_state_and_failure_class(): assert sample("litellm_redis_circuit_breaker_state", {"state": "open"}) == open_gauge_before + 1 assert sample("litellm_redis_circuit_breaker_state", {"state": "closed"}) == closed_gauge_before + breaker._opened_at = time.time() - 9999 + assert breaker.is_open() is False breaker.record_success() assert sample("litellm_redis_circuit_breaker_state", {"state": "open"}) == open_gauge_before assert sample("litellm_redis_circuit_breaker_state", {"state": "closed"}) == closed_gauge_before + 1 @@ -1030,3 +1066,40 @@ def test_sync_guard_counts_a_timeout_as_a_timeout(): _run_under_circuit_breaker_sync(breaker, "op", timing_out_call) assert breaker.is_open() is False + + +def test_success_admitted_before_the_breaker_opened_cannot_close_it(): + """A stale in-flight success must not close a breaker that opened while it ran. + + Calls admitted while the breaker was still closed finish after later failures opened it. + Recording their success unconditionally closed the breaker again, skipping the recovery + timeout and the single half-open probe, so the breaker flapped between open and closed + on every straggler while Redis was still down. + """ + from litellm.caching.redis_cache import RedisCircuitBreaker + + breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60) + for _ in range(3): + breaker.record_failure() + assert breaker._state == breaker.OPEN + + breaker.record_success() + + assert breaker._state == breaker.OPEN + assert breaker.is_open() is True + + +def test_recovery_probe_still_closes_the_breaker(): + from litellm.caching.redis_cache import RedisCircuitBreaker + + breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60) + for _ in range(3): + breaker.record_failure() + breaker._opened_at = time.time() - 9999 + assert breaker.is_open() is False + assert breaker._state == breaker.HALF_OPEN + + breaker.record_success() + + assert breaker._state == breaker.CLOSED + assert breaker.is_open() is False diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 8d8bc15f9be..2050e65d2a1 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -2616,6 +2616,28 @@ class TestCLIKeyRegenerationFlow: ) cache.set_cache.assert_not_called() + def test_cli_sso_flow_lookup_treats_an_open_redis_breaker_as_a_miss(self): + """A Redis read refused by the open circuit breaker is a missing session, not a server error. + + The direct Redis read is what keeps the flow authoritative across workers, so the + refusal must not fall back to a possibly stale in-memory copy either. + """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + from litellm.proxy.management_endpoints.ui_sso import _get_cli_sso_flow_or_raise + + redis_cache = MagicMock() + redis_cache.get_cache.side_effect = RedisCircuitBreakerOpenError("Redis circuit breaker is open") + cache = MagicMock() + cache.redis_cache = redis_cache + cache.get_cache.return_value = {"poll_secret_hash": "stale", "sso_complete": False} + + with pytest.raises(HTTPException) as exc_info: + _get_cli_sso_flow_or_raise(login_id="cli-breaker_open_1234567890", cache=cache) + + assert exc_info.value.status_code == 400 + assert "not found or expired" in exc_info.value.detail + cache.get_cache.assert_not_called() + def test_cli_sso_flow_with_enum_survives_redis_round_trip(self): """ RedisCache stores values via str(value) and reads them back through diff --git a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py index c343652efd9..2f47736a398 100644 --- a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py +++ b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py @@ -1146,6 +1146,54 @@ async def test_prepare_window_spend_counter_increment_missing_window_start_inval assert fake_cache.redis_cache.async_increment.called is False +# --------------------------------------------------------------------------- +# _apply_spend_counter_increments +# --------------------------------------------------------------------------- + + +def _two_pending_increments() -> tuple[ps._PendingSpendIncrement, ...]: + return ( + ps._PendingSpendIncrement(counter_key="spend:key:k", increment=1.5), + ps._PendingSpendIncrement(counter_key="spend:team:t", increment=1.5), + ) + + +@pytest.mark.asyncio +async def test_apply_spend_counter_increments_open_breaker_invalidates_and_returns(monkeypatch): + """An open Redis circuit breaker is a known, already-logged state, not a per-request tracking failure. + + Re-raising the refusal sent every request through the cost callback's error path, which + logged an ERROR and fired the failed-tracking alert once per request for the whole outage. + """ + from litellm.caching.redis_cache import RedisCircuitBreakerOpenError + + fake_cache = _make_spend_counter_cache() + fake_cache.redis_cache.async_increment_pipeline = AsyncMock( + side_effect=RedisCircuitBreakerOpenError("Redis circuit breaker is open") + ) + monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) + + await ps._apply_spend_counter_increments(_two_pending_increments()) + + deleted_keys = sorted(call.kwargs["key"] for call in fake_cache.in_memory_cache.delete_cache.call_args_list) + assert deleted_keys == ["spend:key:k", "spend:team:t"] + fake_cache.in_memory_cache.set_cache.assert_not_called() + + +@pytest.mark.asyncio +async def test_apply_spend_counter_increments_other_redis_error_invalidates_and_raises(monkeypatch): + fake_cache = _make_spend_counter_cache() + fake_cache.redis_cache.async_increment_pipeline = AsyncMock(side_effect=ConnectionError("redis down")) + monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) + + with pytest.raises(ConnectionError, match="redis down"): + await ps._apply_spend_counter_increments(_two_pending_increments()) + + deleted_keys = sorted(call.kwargs["key"] for call in fake_cache.in_memory_cache.delete_cache.call_args_list) + assert deleted_keys == ["spend:key:k", "spend:team:t"] + fake_cache.in_memory_cache.set_cache.assert_not_called() + + # --------------------------------------------------------------------------- # _ensure_spend_counter_initialized # --------------------------------------------------------------------------- From 5ea2f9698295a0e53735a9f8a001262de5a7dee6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 17:59:22 -0700 Subject: [PATCH 392/411] feat(ui): link the Organization cell on the Virtual Keys page too Same treatment as User, Team and Created By in the previous commit: the Organization column rendered the alias as dead text, so it now goes through IdentityCell with an orgDetailHref. Claude-Session: https://claude.ai/code/session_01NfwfQhamRNnSqgXMUjf3h4 --- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 19 +++++++++++++++++-- .../VirtualKeysPage/keyTableColumns.tsx | 12 ++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 57786d71c66..c303dba697b 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -476,12 +476,14 @@ it("should display 'Default Proxy Admin' for user_id when value is 'default_user describe("entity links out of the key rows", () => { const keyRow = async () => (await screen.findByText("Test Key Alias")).closest("tr") as HTMLElement; - const enableCreatedByColumn = async (user: ReturnType) => { + const enableColumn = async (user: ReturnType, title: string) => { await user.click(screen.getByRole("button", { name: "Columns" })); - await user.click(await screen.findByText("Created By")); + await user.click(await screen.findByText(title)); await user.keyboard("{Escape}"); }; + const enableCreatedByColumn = (user: ReturnType) => enableColumn(user, "Created By"); + it("points the User and Team cells at their detail pages", async () => { renderWithProviders(); @@ -493,6 +495,19 @@ describe("entity links out of the key rows", () => { expect(within(row).getByRole("link", { name: "Test Team" })).toHaveAttribute("href", "/ui/teams?team=team-1"); }); + it("points the Organization cell at the org's detail page", async () => { + mockUseKeys.mockReturnValue(keysResult([{ ...mockKey, org_id: "org-1" }])); + const user = userEvent.setup(); + renderWithProviders(); + await enableColumn(user, "Organization"); + + const row = await keyRow(); + expect(within(row).getByRole("link", { name: "Test Organization" })).toHaveAttribute( + "href", + "/ui/organizations?org=org-1", + ); + }); + it("points the Created By cell at the creator's detail page", async () => { mockUseKeys.mockReturnValue( keysResult([ diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index c87bc90fb47..9dd4b1c2d59 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -16,7 +16,7 @@ import { StatusBadge, type StatusTone, } from "@/components/shared/table_cells"; -import { teamDetailHref, userDetailHref } from "@/utils/entityLinks"; +import { orgDetailHref, teamDetailHref, userDetailHref } from "@/utils/entityLinks"; import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; @@ -218,12 +218,12 @@ export const getKeyTableColumns = ({ const orgId = info.getValue() as string | null; if (!orgId) return "-"; const org = organizations.find((o) => o.organization_id === orgId); - const displayValue = org?.organization_alias || orgId; - const width = info.cell.column.getSize(); return ( - - {displayValue} - + ); }, }, From d1a1cda144391eefecd21ecc6d8d096af04f8782 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 10 Sep 2026 18:07:12 -0700 Subject: [PATCH 393/411] test: respect optional logging payload fields --- tests/local_testing/test_custom_callback_input.py | 4 ++-- tests/logging_callback_tests/test_datadog.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py index f0f24a6e6b2..834570091bd 100644 --- a/tests/local_testing/test_custom_callback_input.py +++ b/tests/local_testing/test_custom_callback_input.py @@ -1076,7 +1076,7 @@ def test_standard_logging_payload(model, turn_off_message_logging): ) ) - keys_list = list(StandardLoggingPayload.__annotations__.keys()) + keys_list = list(StandardLoggingPayload.__required_keys__) for k in keys_list: assert ( @@ -1190,7 +1190,7 @@ def test_standard_logging_payload_audio(turn_off_message_logging, stream): ) ) - keys_list = list(StandardLoggingPayload.__annotations__.keys()) + keys_list = list(StandardLoggingPayload.__required_keys__) for k in keys_list: assert ( diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index 83a652e8884..7ac9ac0b5ad 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -270,7 +270,7 @@ async def test_datadog_logging_http_request(): message = json.loads(body[0]["message"]) print("logged message", json.dumps(message, indent=4)) - expected_message_fields = StandardLoggingPayload.__annotations__.keys() + expected_message_fields = StandardLoggingPayload.__required_keys__ for field in expected_message_fields: assert field in message, f"Field '{field}' is missing from the message" From 4fbe2276a1ece2f58d047c60660cfe630fdbbede Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:15:24 -0700 Subject: [PATCH 394/411] fix(logging): finish response metadata before the sync logging thread reads it (#39869) * fix(logging): finish response metadata before the sync logging thread reads it The async and sync client wrappers handed the response to the threaded success handler before computing its cost, call id, and api_base, so that thread inserted into the same metadata dict the request coroutine was still iterating and a finished chat completion turned into a 500 (dictionary changed size during iteration). Metadata is now finalized first, and the merge and header copies snapshot their dicts before iterating. * fix(logging): snapshot metadata with a dict copy and drop redundant comment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): copy metadata via dict.copy and dedupe Final import Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 39 +++--- litellm/utils.py | 112 +++++++++++------- .../test_litellm_logging.py | 59 +++++++++ tests/test_litellm/test_utils.py | 46 +++++++ 4 files changed, 193 insertions(+), 63 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 893bfeefff8..6ee68ab21c5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -295,6 +295,9 @@ def _get_provider_request_id(original_exception: Exception) -> str | None: # Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys _CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) _MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS +_UNSERIALIZABLE_METADATA_KEYS: Final[frozenset[str]] = frozenset( + ("user_api_key_auth", "user_api_key_budget_reservation") +) sentry_sdk_instance = None capture_exception = None @@ -5386,23 +5389,23 @@ class StandardLoggingPayloadSetup: Returns: dict: Merged metadata with user API key fields taking precedence """ - merged_metadata: Final[dict] = {} - - # Start with metadata (user API key fields) - but skip non-serializable objects - if litellm_params.get("metadata") and isinstance(litellm_params.get("metadata"), dict): - for key, value in litellm_params["metadata"].items(): - # Skip non-serializable objects like UserAPIKeyAuth - if key in {"user_api_key_auth", "user_api_key_budget_reservation"}: - continue - merged_metadata[key] = value - - # Then merge litellm_metadata (model-related fields) - this will NOT overwrite existing keys - if litellm_params.get("litellm_metadata") and isinstance(litellm_params.get("litellm_metadata"), dict): - for key, value in litellm_params["litellm_metadata"].items(): - if key not in merged_metadata: # Don't overwrite existing keys from metadata - merged_metadata[key] = value - - return merged_metadata + metadata: Final = litellm_params.get("metadata") + litellm_metadata: Final = litellm_params.get("litellm_metadata") + user_metadata: Final = MappingProxyType( + { + key: value + for key, value in (metadata.copy().items() if isinstance(metadata, dict) else ()) + if key not in _UNSERIALIZABLE_METADATA_KEYS + } + ) + model_metadata: Final = MappingProxyType( + { + key: value + for key, value in (litellm_metadata.copy().items() if isinstance(litellm_metadata, dict) else ()) + if key not in user_metadata + } + ) + return {**user_metadata, **model_metadata} # mutable-ok: function contract returns a plain dict @staticmethod def get_standard_logging_metadata( @@ -5660,7 +5663,7 @@ class StandardLoggingPayloadSetup: additional_logging_headers[key] = additiona_headers[_key] # Preserve all remaining headers verbatim (e.g. llm_provider-x-request-id) - for k, v in additiona_headers.items(): + for k, v in additiona_headers.copy().items(): if k.lower() not in typed_keys: additional_logging_headers[k] = v diff --git a/litellm/utils.py b/litellm/utils.py index b93938a480e..a765e1b1246 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1196,6 +1196,47 @@ def function_setup( raise e +def _dispatch_success_logging( + logging_obj: LiteLLMLoggingObject, + result: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + is_completion_with_fallbacks: bool, + is_litellm_internal_call: bool, +) -> None: + if not is_litellm_internal_call: + if getattr(logging_obj, "_defer_async_logging", False): + + def _enqueue_deferred_logging() -> None: + asyncio.create_task( + _client_async_logging_helper( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + ) + ) + + logging_obj._enqueue_deferred_logging = _enqueue_deferred_logging + else: + asyncio.create_task( + _client_async_logging_helper( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + ) + ) + + logging_obj.handle_sync_success_callbacks_for_async_calls( + result=result, + start_time=start_time, + end_time=end_time, + ) + + async def _client_async_logging_helper( logging_obj: LiteLLMLoggingObject, result, @@ -1663,6 +1704,16 @@ def client(original_function): kwargs=kwargs, ) + _update_response_metadata: Final = getattr(sys.modules[__name__], "update_response_metadata") + _update_response_metadata( + result=result, + logging_obj=logging_obj, + model=model, + kwargs=kwargs, + start_time=start_time, + end_time=end_time, + ) + # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated verbose_logger.info("Wrapper: Completed Call, calling success_handler") # Copy the current context to propagate it to the background thread @@ -1677,15 +1728,6 @@ def client(original_function): end_time, ) # RETURN RESULT - update_response_metadata = getattr(sys.modules[__name__], "update_response_metadata") - update_response_metadata( - result=result, - logging_obj=logging_obj, - model=model, - kwargs=kwargs, - start_time=start_time, - end_time=end_time, - ) return result except Exception as e: call_type = original_function.__name__ @@ -1945,48 +1987,20 @@ def client(original_function): args=args, ) - # LOG SUCCESS - handle streaming success logging in the _next_ object - # Internal sub-calls (e.g. emulated file-search steps) share the - # parent's logging obj; skip async logging here so only the outer call bills once. - # NOTE: streaming requests return early (before this point) via - # CustomStreamWrapper, so this block is non-streaming only. - if not _is_litellm_internal_call: - if getattr(logging_obj, "_defer_async_logging", False): - - def _enqueue_deferred_logging() -> None: - asyncio.create_task( - _client_async_logging_helper( - logging_obj=logging_obj, - result=result, - start_time=start_time, - end_time=end_time, - is_completion_with_fallbacks=is_completion_with_fallbacks, - ) - ) - - logging_obj._enqueue_deferred_logging = _enqueue_deferred_logging - else: - asyncio.create_task( - _client_async_logging_helper( - logging_obj=logging_obj, - result=result, - start_time=start_time, - end_time=end_time, - is_completion_with_fallbacks=is_completion_with_fallbacks, - ) - ) - - logging_obj.handle_sync_success_callbacks_for_async_calls( - result=result, - start_time=start_time, - end_time=end_time, - ) # REBUILD EMBEDDING CACHING if ( isinstance(result, EmbeddingResponse) and _caching_handler_response is not None and _caching_handler_response.final_embedding_cached_response is not None ): + _dispatch_success_logging( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + is_litellm_internal_call=_is_litellm_internal_call, + ) return _llm_caching_handler._combine_cached_embedding_response_with_api_result( _caching_handler_response=_caching_handler_response, embedding_response=result, @@ -2002,6 +2016,14 @@ def client(original_function): start_time=start_time, end_time=end_time, ) + _dispatch_success_logging( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + is_litellm_internal_call=_is_litellm_internal_call, + ) return result except Exception as e: diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index aeaa8f18c26..2f6339dcdbb 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3,6 +3,7 @@ import contextlib import datetime import os import sys +from collections.abc import Callable from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -6945,3 +6946,61 @@ def test_classifier_audit_is_not_added_to_other_calls(logging_obj, call_type, or logging_obj.model_call_details["litellm_params"] = {"metadata": {"internal_call_origin": origin}} logging_obj.pre_call(input=[], api_key=None, additional_args={"complete_input_dict": {"input": "embedding"}}) assert logging_obj.classifier_input is None + + +def _run_while_a_thread_grows(target: dict, read: Callable[[], None], reads: int) -> None: + import itertools + import threading + + stop: Final = threading.Event() + + def grow() -> None: + for counter in itertools.count(): + if stop.is_set(): + return + key: Final = f"late_{counter % 64}" + if key in target: + del target[key] + else: + target[key] = counter + + writer: Final = threading.Thread(target=grow, daemon=True) + previous_interval: Final = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + writer.start() + try: + for _ in range(reads): + read() + finally: + stop.set() + writer.join(timeout=5) + sys.setswitchinterval(previous_interval) + + +def test_merge_litellm_metadata_survives_a_thread_growing_metadata_mid_merge(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + metadata: Final = {f"key_{i}": i for i in range(2000)} + litellm_params: Final = {"metadata": metadata, "litellm_metadata": {"model_group": "gpt"}} + + def read() -> None: + merged: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) + assert merged["key_1999"] == 1999 + assert merged["model_group"] == "gpt" + + _run_while_a_thread_grows(metadata, read, reads=300) + + +def test_get_additional_headers_survives_a_thread_growing_headers_mid_copy(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + headers: Final = {f"llm_provider-x-custom-{i}": str(i) for i in range(2000)} + headers["x-ratelimit-remaining-requests"] = "7" + + def read() -> None: + copied: Final = StandardLoggingPayloadSetup.get_additional_headers(headers) + assert copied is not None + assert copied["x_ratelimit_remaining_requests"] == 7 + assert copied["llm_provider-x-custom-1999"] == "1999" + + _run_while_a_thread_grows(headers, read, reads=300) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 2bdde43ff2d..7753cb7d770 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1,9 +1,12 @@ import asyncio +import contextlib import json import logging import os +import queue import threading from datetime import datetime, timedelta, timezone +from collections.abc import Iterator from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -6298,3 +6301,46 @@ async def test_async_mock_completion_streaming_obj_raises_mock_exception_before_ ) with pytest.raises(litellm.MockException): await _async_mock_stream_snapshots(mock_exception, 51234) + + + +@contextlib.contextmanager +def _recording_hidden_params_at_submit(submit_target: str) -> "Iterator[queue.SimpleQueue[dict[str, object]]]": + seen: Final = queue.SimpleQueue() + + def record_submit(_fn, *args, **_kwargs): + response: Final = next(arg for arg in args if isinstance(arg, litellm.ModelResponse)) + seen.put(dict(response._hidden_params)) + return MagicMock() + + with patch(submit_target, side_effect=record_submit): + yield seen + + +@pytest.mark.asyncio +async def test_acompletion_finishes_response_metadata_before_handing_the_response_to_the_logging_thread(monkeypatch): + monkeypatch.setattr(litellm, "success_callback", [lambda kwargs, response, start_time, end_time: None]) + with _recording_hidden_params_at_submit("litellm.litellm_core_utils.litellm_logging.executor.submit") as seen: + await litellm.acompletion( + model="gpt-5.5", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello there!", + num_retries=0, + ) + snapshot: Final = seen.get_nowait() + assert snapshot["litellm_call_id"] + assert snapshot["response_cost"] is not None + assert snapshot["api_base"] + + +def test_completion_finishes_response_metadata_before_handing_the_response_to_the_logging_thread(): + with _recording_hidden_params_at_submit("litellm.utils.executor.submit") as seen: + litellm.completion( + model="gpt-5.5", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello there!", + ) + snapshot: Final = seen.get_nowait() + assert snapshot["litellm_call_id"] + assert snapshot["response_cost"] is not None + assert snapshot["api_base"] From 7169ddaef6ad4847f630f2c2bcefa2799db19c92 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:16:59 -0700 Subject: [PATCH 395/411] fix(router): treat a breaker-refused Redis read as a miss in the health state cache The sync Redis read now raises while the circuit breaker is open, and the health state merge caught that as a generic error, skipping the local write and logging an error on every background health check cycle. Read the shared snapshot through a helper that treats the refused read as a miss so the merge falls back to the pod-local copy the way a swallowed connection error already did --- litellm/router_utils/health_state_cache.py | 20 ++++++++++---- .../router_utils/test_health_state_cache.py | 27 +++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/litellm/router_utils/health_state_cache.py b/litellm/router_utils/health_state_cache.py index 22d816e13e9..c8ca7105392 100644 --- a/litellm/router_utils/health_state_cache.py +++ b/litellm/router_utils/health_state_cache.py @@ -12,6 +12,7 @@ from typing_extensions import TypedDict from litellm import verbose_logger from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -27,6 +28,16 @@ class DeploymentHealthStateValue(TypedDict): reason: str +def _read_shared_health_snapshot(cache: DualCache, key: str) -> object: + redis_cache: Final = cache.redis_cache + if redis_cache is None: + return None + try: + return redis_cache.get_cache(key) + except RedisCircuitBreakerOpenError: + return None + + class DeploymentHealthCache: """ Cache for deployment health states produced by background health checks. @@ -50,13 +61,12 @@ class DeploymentHealthCache: coexist on the one shared entry without erasing each other's results. The snapshot is read from Redis when available, since a pod-local read would only ever see this writer's own previous merge. When the Redis - read comes back empty (a miss, or a swallowed connection error), the - pod-local copy of the last merge is used so peers are not erased. + read comes back empty (a miss, a swallowed connection error, or a read + refused by the open circuit breaker), the pod-local copy of the last + merge is used so peers are not erased. """ try: - redis_raw: Final = ( - self.cache.redis_cache.get_cache(self.CACHE_KEY) if self.cache.redis_cache is not None else None - ) + redis_raw: Final = _read_shared_health_snapshot(self.cache, self.CACHE_KEY) raw: Final = redis_raw if isinstance(redis_raw, dict) else self.cache.get_cache(key=self.CACHE_KEY) existing: Final = raw if isinstance(raw, dict) else {} expiry_seconds: Final = self.staleness_threshold * 1.5 diff --git a/tests/test_litellm/router_utils/test_health_state_cache.py b/tests/test_litellm/router_utils/test_health_state_cache.py index ffd031f9b7d..aa976bb1002 100644 --- a/tests/test_litellm/router_utils/test_health_state_cache.py +++ b/tests/test_litellm/router_utils/test_health_state_cache.py @@ -7,6 +7,7 @@ import time import pytest from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.router_utils.health_state_cache import DeploymentHealthCache @@ -145,8 +146,11 @@ class _SharedRedisFake: def __init__(self): self.store = {} self.fail_get = False + self.breaker_open = False def get_cache(self, key, parent_otel_span=None, **kwargs): + if self.breaker_open: + raise RedisCircuitBreakerOpenError("Redis circuit breaker is open - skipping get_cache") if self.fail_get: return None # RedisCache.get_cache swallows connection errors and returns None return self.store.get(key) @@ -192,3 +196,26 @@ def test_failed_redis_read_falls_back_to_local_copy(): {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} ) assert set(redis_fake.store[DeploymentHealthCache.CACHE_KEY]) == {"prod-bad", "internal-bad"} + + +def test_open_circuit_breaker_read_still_merges_into_local_copy(caplog): + """A read refused by the open breaker is a miss, so the merge and local write still happen quietly.""" + redis_fake = _SharedRedisFake() + pod_a = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0) + pod_b = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0) + pod_a.set_deployment_health_states( + {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + pod_b.set_deployment_health_states( + {"internal-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "timeout"}} + ) + pod_a.set_deployment_health_states( + {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + redis_fake.breaker_open = True + with caplog.at_level("ERROR"): + pod_a.set_deployment_health_states( + {"prod-new-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}} + ) + assert caplog.records == [] + assert pod_a.get_unhealthy_deployment_ids() == {"prod-bad", "internal-bad", "prod-new-bad"} From dcdd884352b1f39f51dd0719bba4eca0046859f8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:36:19 -0700 Subject: [PATCH 396/411] fix(caching): let only the recovery probe close a half-open Redis breaker A call admitted before the breaker opened could finish while the breaker was HALF_OPEN and close it before the designated probe reported, so Redis traffic resumed on a stale answer. The admission now records whether the call is the probe and only the probe's success closes a half-open breaker. The cron job lock manager also logged an error every cycle the open breaker refused its Redis call, one line per job per pod. That refusal is now a debug line like every other guarded call, while real Redis errors still log at error --- litellm/caching/redis_cache.py | 43 ++++++++++------ .../db_transaction_queue/pod_lock_manager.py | 7 +-- .../test_litellm/caching/test_redis_cache.py | 49 +++++++++++++++++++ .../test_pod_lock_manager.py | 18 +++++++ 4 files changed, 100 insertions(+), 17 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 5b1a9739eb8..812b435ae5f 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -18,6 +18,7 @@ import logging import time from collections.abc import Awaitable, Callable, Sequence from contextvars import ContextVar +from dataclasses import dataclass from datetime import timedelta from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast @@ -198,6 +199,9 @@ class RedisCircuitBreaker: self._state = self.CLOSED _breaker_metrics().record_state_change(None, self._state) + def is_half_open(self) -> bool: + return self._state == self.HALF_OPEN + def is_open(self) -> bool: """Returns True if Redis calls should be skipped.""" if not self.enabled: @@ -405,21 +409,32 @@ def log_redis_failure( logger.log(level, "%s: %s", message, exc, exc_info=exc if with_traceback else None) -def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> int: - """Reject the call if the breaker is open, else return the swallowed-failure count to compare against.""" +@dataclass(frozen=True, slots=True) +class _BreakerAdmission: + swallowed_before: int + is_probe: bool + + +def _enter_circuit_breaker(breaker: RedisCircuitBreaker, name: str) -> _BreakerAdmission: + """Reject the call if the breaker is open, else record what its success may later prove.""" if breaker.is_open(): raise RedisCircuitBreakerOpenError(f"Redis circuit breaker is open — skipping {name}") - return _swallowed_redis_failures.get() + return _BreakerAdmission(swallowed_before=_swallowed_redis_failures.get(), is_probe=breaker.is_half_open()) -def _exit_circuit_breaker(breaker: RedisCircuitBreaker, swallowed_before: int) -> None: - """Record success only when nothing failed while the call ran. +def _exit_circuit_breaker(breaker: RedisCircuitBreaker, admission: _BreakerAdmission) -> None: + """Record success only when nothing failed while the call ran and the call may vouch for Redis. Several Redis methods catch their own connection errors and return a default, so a - method that returned is not on its own proof of a healthy Redis. + method that returned is not on its own proof of a healthy Redis. While the breaker is + half open only the designated recovery probe may close it: a call admitted before the + breaker opened that finishes late says nothing about whether Redis recovered. """ - if _swallowed_redis_failures.get() == swallowed_before: - breaker.record_success() + if _swallowed_redis_failures.get() != admission.swallowed_before: + return + if breaker.is_half_open() and not admission.is_probe: + return + breaker.record_success() async def _run_under_circuit_breaker( @@ -432,14 +447,14 @@ async def _run_under_circuit_breaker( Shared by the method decorator and the Lua script executor so both feed the same health signal. """ - swallowed_before: Final = _enter_circuit_breaker(breaker, name) + admission: Final = _enter_circuit_breaker(breaker, name) try: result: Final = await call() except Exception as e: if _is_redis_health_failure(e): breaker.record_failure(is_timeout=_is_redis_timeout_failure(e)) raise - _exit_circuit_breaker(breaker, swallowed_before) + _exit_circuit_breaker(breaker, admission) return result @@ -449,14 +464,14 @@ def _run_under_circuit_breaker_sync( call: Callable[[], _RedisCallResult], ) -> _RedisCallResult: """Run one blocking Redis call under a circuit breaker, feeding the same health signal as the async path.""" - swallowed_before: Final = _enter_circuit_breaker(breaker, name) + admission: Final = _enter_circuit_breaker(breaker, name) try: result: Final = call() except Exception as e: if _is_redis_health_failure(e): breaker.record_failure(is_timeout=_is_redis_timeout_failure(e)) raise - _exit_circuit_breaker(breaker, swallowed_before) + _exit_circuit_breaker(breaker, admission) return result @@ -1395,12 +1410,12 @@ class RedisCache(BaseCache): key_value_dict = {} _key_list: Final = [key for key in key_list if key is not None] start_time: Final = time.time() - swallowed_before: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") + admission: Final = _enter_circuit_breaker(self._circuit_breaker, "batch_get_cache") try: _keys: Final = [self.check_and_fix_namespace(key=cache_key or "") for cache_key in _key_list] results: Final = self._run_redis_mget_operation(keys=_keys) - _exit_circuit_breaker(self._circuit_breaker, swallowed_before) + _exit_circuit_breaker(self._circuit_breaker, admission) end_time: Final = time.time() _duration: Final = end_time - start_time self.service_logger_obj.service_success_hook( diff --git a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py index 4be1331e955..bc67617e444 100644 --- a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py +++ b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py @@ -1,10 +1,11 @@ import asyncio import json +import logging from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid -from litellm.caching.redis_cache import RedisCache +from litellm.caching.redis_cache import RedisCache, log_redis_failure from litellm.constants import DEFAULT_CRON_JOB_LOCK_TTL_SECONDS from litellm.proxy.db.db_transaction_queue.base_update_queue import service_logger_obj from litellm.types.services import ServiceTypes @@ -109,7 +110,7 @@ end ) return False except Exception as e: - verbose_proxy_logger.error("Error acquiring Redis lock for %s: %s", cronjob_id, e) + log_redis_failure(verbose_proxy_logger, logging.ERROR, f"Error acquiring Redis lock for {cronjob_id}", e) return False async def release_lock( @@ -151,7 +152,7 @@ end cronjob_id, ) except Exception as e: - verbose_proxy_logger.error("Error releasing Redis lock for %s: %s", cronjob_id, e) + log_redis_failure(verbose_proxy_logger, logging.ERROR, f"Error releasing Redis lock for {cronjob_id}", e) async def _compare_and_delete_lock(self, lock_key: str) -> int: """ diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index c6f0caa1dca..9df9c44a30f 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1103,3 +1103,52 @@ def test_recovery_probe_still_closes_the_breaker(): assert breaker._state == breaker.CLOSED assert breaker.is_open() is False + + +@pytest.mark.asyncio +async def test_stale_success_during_the_recovery_probe_leaves_the_breaker_to_the_probe(): + """A call admitted before the trip that finishes while HALF_OPEN must not close the breaker. + + Only the one call designated as the recovery probe has actually reached Redis after the + outage, so closing on the straggler's success resumed full Redis traffic before the probe + had proven anything. + """ + from litellm.caching.redis_cache import RedisCircuitBreaker, _run_under_circuit_breaker + + breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60) + stale_admitted = asyncio.Event() + stale_release = asyncio.Event() + probe_admitted = asyncio.Event() + probe_release = asyncio.Event() + + async def stale_call() -> str: + stale_admitted.set() + await stale_release.wait() + return "stale" + + async def probe_call() -> str: + probe_admitted.set() + await probe_release.wait() + return "probe" + + stale = asyncio.ensure_future(_run_under_circuit_breaker(breaker, "op", stale_call)) + await stale_admitted.wait() + for _ in range(3): + breaker.record_failure() + assert breaker._state == breaker.OPEN + breaker._opened_at = time.time() - 9999 + probe = asyncio.ensure_future(_run_under_circuit_breaker(breaker, "op", probe_call)) + await probe_admitted.wait() + assert breaker._state == breaker.HALF_OPEN + + stale_release.set() + assert await stale == "stale" + + assert breaker._state == breaker.HALF_OPEN, "the straggler must not close the breaker for the probe" + assert breaker.is_open() is True + + probe_release.set() + assert await probe == "probe" + + assert breaker._state == breaker.CLOSED + assert breaker.is_open() is False diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py index ecd5c5f50c0..4684c3213d6 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py @@ -1,4 +1,5 @@ import json +import logging from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -6,6 +7,7 @@ import pytest from fastapi.testclient import TestClient +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import DEFAULT_CRON_JOB_LOCK_TTL_SECONDS from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager @@ -215,6 +217,22 @@ async def test_redis_error_handling(pod_lock_manager, mock_redis): ) +@pytest.mark.asyncio +async def test_lock_refused_by_the_open_circuit_breaker_is_not_logged_as_an_error(pod_lock_manager, mock_redis, caplog): + """Every cron job retries its lock on a timer, so an open breaker must not add an error line per cycle.""" + refused = RedisCircuitBreakerOpenError("Redis circuit breaker is open - skipping async_set_cache") + mock_redis.async_set_cache.side_effect = refused + mock_redis.async_get_cache.return_value = pod_lock_manager.pod_id + mock_redis.async_delete_cache.side_effect = refused + + with caplog.at_level(logging.ERROR): + acquired = await pod_lock_manager.acquire_lock(cronjob_id="test_job") + await pod_lock_manager.release_lock(cronjob_id="test_job") + + assert acquired is False + assert caplog.records == [] + + @pytest.mark.asyncio async def test_bytes_handling(pod_lock_manager, mock_redis): """ From 880ccc76a5bcc89b07a7ca337662a902bf56083e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:37:39 -0700 Subject: [PATCH 397/411] fix(streaming): keep admitted mock streams alive with empty stream_options and honor zero prompt counts (#40650) * fix(streaming): keep usage-only chunks from crashing streams with empty stream_options The usage-only chunk branch in CustomStreamWrapper.chunk_creator indexed stream_options["include_usage"] directly, so a caller passing stream_options={} hit a KeyError that surfaced as MidStreamFallbackError. Streaming mock_response with an admission input_tokens count (#40637) now always emits such a chunk, which made the crash reachable. Reuse the send_stream_usage policy computed at init instead. Also annotate the #40637 test bindings with Final. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(streaming): report admitted zero prompt tokens instead of recounting in mock streams Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/streaming_handler.py | 7 +- litellm/main.py | 9 ++ tests/test_litellm/test_main.py | 143 ++++++++++++++---- 3 files changed, 131 insertions(+), 28 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index db23929e0c3..6a5a8832cc6 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -236,9 +236,11 @@ class CustomStreamWrapper: stream_options=None, make_call: Callable | None = None, _response_headers: dict | httpx.Headers | None = None, + count_prompt_tokens: Callable[[], int] | None = None, ): self.model = model self.make_call = make_call + self.count_prompt_tokens = count_prompt_tokens self.custom_llm_provider = custom_llm_provider self.logging_obj: LiteLLMLoggingObject = logging_obj self.completion_stream = completion_stream @@ -1641,7 +1643,7 @@ class CustomStreamWrapper: except Exception: model_response.choices[0].delta = Delta() else: - if self.stream_options is not None and self.stream_options["include_usage"] is True: + if self.send_stream_usage is True: model_response.choices = [] return model_response self._record_usage_only_chunk(model_response=model_response) @@ -1996,6 +1998,7 @@ class CustomStreamWrapper: chunks=self.chunks, messages=self.messages, logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, ) except Exception as e: # stream_chunk_builder can re-raise (as APIError) on large agentic @@ -2248,6 +2251,7 @@ class CustomStreamWrapper: chunks=self.chunks, messages=self.messages, logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, ) except Exception as e: # see sync __next__: a raise from stream_chunk_builder inside this @@ -2371,6 +2375,7 @@ class CustomStreamWrapper: chunks=self.chunks, messages=self.messages if isinstance(self.messages, list) else None, logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, ) if partial_response is None: return diff --git a/litellm/main.py b/litellm/main.py index 10fb32828f1..17edafcdfca 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -850,6 +850,12 @@ def admission_input_tokens(kwargs: Mapping[str, object]) -> int | None: ) +def admitted_prompt_token_counter(prompt_tokens: int | None) -> Callable[[], int] | None: + if prompt_tokens is None: + return None + return lambda: prompt_tokens + + def mock_completion( model: str, messages: list, @@ -935,6 +941,7 @@ def mock_completion( if stream is True: model_response = ModelResponseStream() + count_prompt_tokens: Final = admitted_prompt_token_counter(prompt_tokens) # don't try to access stream object, if kwargs.get("acompletion", False) is True: return CustomStreamWrapper( @@ -944,6 +951,7 @@ def mock_completion( model=model, custom_llm_provider="openai", logging_obj=logging, + count_prompt_tokens=count_prompt_tokens, ) return CustomStreamWrapper( completion_stream=mock_completion_streaming_obj( @@ -952,6 +960,7 @@ def mock_completion( model=model, custom_llm_provider="openai", logging_obj=logging, + count_prompt_tokens=count_prompt_tokens, ) if isinstance(mock_response, litellm.MockException): raise mock_response diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index a36ca229981..f71225c6fc5 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2422,9 +2422,13 @@ def test_mock_completion_usage_falls_back_to_default_without_admission_count(): _ADMISSION_INPUT_TOKENS: Final = 51234 -_ADMISSION_METADATA: Final = { - "user_api_key_budget_reservation": {"reserved_cost": 1.0, "input_tokens": _ADMISSION_INPUT_TOKENS} -} + + +def _admission_metadata(input_tokens: int) -> dict[str, object]: # mutable-ok: logging writes into metadata + return {"user_api_key_budget_reservation": {"reserved_cost": 1.0, "input_tokens": input_tokens}} + + +_ADMISSION_METADATA: Final = _admission_metadata(_ADMISSION_INPUT_TOKENS) _MOCK_STREAM_MESSAGES: Final = [{"role": "user", "content": "hello " * 200}] _STREAM_CHUNK_BUILDER_TOKEN_COUNTER: Final = "litellm.litellm_core_utils.streaming_chunk_builder_utils.token_counter" @@ -2440,7 +2444,7 @@ def _client_usage_chunks(chunks: list[ModelResponseStream]) -> list[Usage]: @pytest.mark.parametrize("n", (None, 2)) def test_mock_completion_stream_usage_reports_admission_input_tokens_without_tokenizer_fallback(n: int | None): with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - chunks = list( + chunks: Final = list( litellm.completion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, @@ -2453,7 +2457,7 @@ def test_mock_completion_stream_usage_reports_admission_input_tokens_without_tok ) ) - usage_chunks = _client_usage_chunks(chunks) + usage_chunks: Final = _client_usage_chunks(chunks) assert len(usage_chunks) == 1 assert usage_chunks[0].prompt_tokens == _ADMISSION_INPUT_TOKENS assert usage_chunks[0].completion_tokens == litellm_main.DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT @@ -2469,7 +2473,7 @@ async def test_mock_acompletion_stream_usage_reports_admission_input_tokens_with n: int | None, ): with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - response = await litellm.acompletion( + response: Final = await litellm.acompletion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, mock_response="ok", @@ -2479,9 +2483,9 @@ async def test_mock_acompletion_stream_usage_reports_admission_input_tokens_with stream_options={"include_usage": True}, litellm_metadata=_ADMISSION_METADATA, ) - chunks = [chunk async for chunk in response] + chunks: Final = [chunk async for chunk in response] - usage_chunks = _client_usage_chunks(chunks) + usage_chunks: Final = _client_usage_chunks(chunks) assert len(usage_chunks) == 1 assert usage_chunks[0].prompt_tokens == _ADMISSION_INPUT_TOKENS assert usage_chunks[0].total_tokens == _ADMISSION_INPUT_TOKENS + usage_chunks[0].completion_tokens @@ -2492,7 +2496,7 @@ async def test_mock_acompletion_stream_usage_reports_admission_input_tokens_with def test_mock_completion_stream_without_include_usage_hides_usage_chunk_but_logs_admission_count(): with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - chunks = list( + chunks: Final = list( litellm.completion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, @@ -2509,10 +2513,48 @@ def test_mock_completion_stream_without_include_usage_hides_usage_chunk_but_logs assert _prompt_token_counter_calls(token_counter) == [] +def test_mock_completion_stream_with_empty_stream_options_completes_and_logs_admission_count(): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks: Final = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={}, + metadata=_ADMISSION_METADATA, + ) + ) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "ok" + assert _client_usage_chunks(chunks) == [] + assert _prompt_token_counter_calls(token_counter) == [] + + +@pytest.mark.asyncio +async def test_mock_acompletion_stream_with_empty_stream_options_completes_and_logs_admission_count(): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + response: Final = await litellm.acompletion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={}, + litellm_metadata=_ADMISSION_METADATA, + ) + chunks: Final = [chunk async for chunk in response] + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "ok" + assert _client_usage_chunks(chunks) == [] + assert _prompt_token_counter_calls(token_counter) == [] + + def test_mock_completion_stream_without_admission_count_falls_back_to_tokenizer(): expected_prompt_tokens: Final = litellm.token_counter(model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES) with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - chunks = list( + chunks: Final = list( litellm.completion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, @@ -2524,7 +2566,7 @@ def test_mock_completion_stream_without_admission_count_falls_back_to_tokenizer( ) ) - usage_chunks = _client_usage_chunks(chunks) + usage_chunks: Final = _client_usage_chunks(chunks) assert len(usage_chunks) == 1 assert usage_chunks[0].prompt_tokens == expected_prompt_tokens assert usage_chunks[0].total_tokens == expected_prompt_tokens + usage_chunks[0].completion_tokens @@ -2535,7 +2577,7 @@ def test_mock_completion_stream_without_admission_count_falls_back_to_tokenizer( async def test_mock_acompletion_stream_without_admission_count_falls_back_to_tokenizer(): expected_prompt_tokens: Final = litellm.token_counter(model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES) with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: - response = await litellm.acompletion( + response: Final = await litellm.acompletion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, mock_response="ok", @@ -2543,40 +2585,87 @@ async def test_mock_acompletion_stream_without_admission_count_falls_back_to_tok stream=True, stream_options={"include_usage": True}, ) - chunks = [chunk async for chunk in response] + chunks: Final = [chunk async for chunk in response] - usage_chunks = _client_usage_chunks(chunks) + usage_chunks: Final = _client_usage_chunks(chunks) assert len(usage_chunks) == 1 assert usage_chunks[0].prompt_tokens == expected_prompt_tokens assert len(_prompt_token_counter_calls(token_counter)) >= 1 -def test_mock_completion_stream_and_non_stream_report_the_same_admission_usage(): - non_stream = litellm.completion( +def _usage_triple(usage: Usage) -> tuple[int, int, int]: + return (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) + + +@pytest.mark.parametrize("input_tokens", (_ADMISSION_INPUT_TOKENS, 0)) +def test_mock_completion_stream_and_non_stream_report_the_same_admission_usage(input_tokens: int): + metadata: Final = _admission_metadata(input_tokens) + non_stream: Final = litellm.completion( model="openai/gpt-5.4-mini", messages=_MOCK_STREAM_MESSAGES, mock_response="ok", api_key="mock", - metadata=_ADMISSION_METADATA, + metadata=metadata, ) - chunks = list( - litellm.completion( + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + chunks: Final = list( + litellm.completion( + model="openai/gpt-5.4-mini", + messages=_MOCK_STREAM_MESSAGES, + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + metadata=metadata, + ) + ) + + assert _usage_triple(non_stream.usage) == _usage_triple(_client_usage_chunks(chunks)[0]) + assert non_stream.usage.prompt_tokens == input_tokens + assert _prompt_token_counter_calls(token_counter) == [] + + +@pytest.mark.asyncio +async def test_mock_acompletion_stream_reports_zero_admission_input_tokens_without_tokenizer_fallback(): + with patch(_STREAM_CHUNK_BUILDER_TOKEN_COUNTER, wraps=litellm.token_counter) as token_counter: + response: Final = await litellm.acompletion( model="openai/gpt-5.4-mini", - messages=_MOCK_STREAM_MESSAGES, + messages=[{"role": "user", "content": ""}], mock_response="ok", api_key="mock", stream=True, stream_options={"include_usage": True}, - metadata=_ADMISSION_METADATA, + litellm_metadata=_admission_metadata(0), + ) + chunks: Final = [chunk async for chunk in response] + + usage_chunks: Final = _client_usage_chunks(chunks) + assert len(usage_chunks) == 1 + assert _usage_triple(usage_chunks[0]) == (0, usage_chunks[0].completion_tokens, usage_chunks[0].completion_tokens) + assert _prompt_token_counter_calls(token_counter) == [] + + +def test_mock_text_completion_stream_and_non_stream_report_the_same_zero_admission_usage(): + metadata: Final = _admission_metadata(0) + non_stream: Final = litellm.text_completion( + model="openai/gpt-5.4-mini", prompt="", mock_response="ok", api_key="mock", metadata=metadata + ) + chunks: Final = list( + litellm.text_completion( + model="openai/gpt-5.4-mini", + prompt="", + mock_response="ok", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + metadata=metadata, ) ) - stream_usage: Final = _client_usage_chunks(chunks)[0] - assert (non_stream.usage.prompt_tokens, non_stream.usage.completion_tokens, non_stream.usage.total_tokens) == ( - stream_usage.prompt_tokens, - stream_usage.completion_tokens, - stream_usage.total_tokens, - ) + stream_usages: Final = tuple(chunk.usage for chunk in chunks if getattr(chunk, "usage", None) is not None) + assert len(stream_usages) == 1 + assert _usage_triple(non_stream.usage) == _usage_triple(stream_usages[0]) + assert non_stream.usage.prompt_tokens == 0 def test_mock_completion_stream_with_model_response(): From 56e2d8846da375ae157e27d1a789030882066f2d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 18:08:13 -0700 Subject: [PATCH 398/411] feat(ui): link the entity cells on the team detail page's keys table The team detail page's Virtual Keys table showed Organization ID, User Email, User ID and Created By as dead text, so getting from a key to the org or user behind it meant copying an id and searching for it. Those four cells now render as links, reusing the sentinel-aware href helpers, so default_user_id and the litellm-dashboard team stay plain text instead of pointing at pages that do not exist. The Created By cell was a verbatim copy of the Virtual Keys page's user popover, so that moved into the shared table_cells kit and both tables now use the one implementation. Claude-Session: https://claude.ai/code/session_01NfwfQhamRNnSqgXMUjf3h4 --- .../VirtualKeysPage/keyTableColumns.tsx | 62 +----------- .../shared/table_cells/UserPopoverCell.tsx | 62 ++++++++++++ .../components/shared/table_cells/index.ts | 1 + .../team/TeamVirtualKeysTable.test.tsx | 62 ++++++++++++ .../components/team/TeamVirtualKeysTable.tsx | 94 ++++++++----------- 5 files changed, 168 insertions(+), 113 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index 9dd4b1c2d59..6eea77ae827 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -9,17 +9,17 @@ import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/h import { Skeleton } from "@/components/ui/skeleton"; import { DateCell, + ENTITY_CELL_TITLE_CLASSES, IdCell, IdentityCell, ModelsCell, SpendBudgetCell, StatusBadge, + UserPopoverCell, type StatusTone, } from "@/components/shared/table_cells"; -import { orgDetailHref, teamDetailHref, userDetailHref } from "@/utils/entityLinks"; -import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; +import { orgDetailHref, teamDetailHref } from "@/utils/entityLinks"; -import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; import { Organization } from "../networking"; @@ -29,8 +29,6 @@ interface KeyStatus { tooltip?: string; } -const ENTITY_CELL_TITLE_CLASSES = "font-mono text-xs font-normal"; - const SPEND_BUDGET_SORT_FIELDS: DataTableSortField[] = [ { id: "spend", label: "Spend" }, { id: "max_budget", label: "Budget" }, @@ -66,60 +64,6 @@ const getKeyStatus = (key: KeyResponse): KeyStatus => { }; }; -const UserPopoverCell = ({ - userAlias, - userEmail, - userId, - width, -}: { - userAlias: string | null; - userEmail: string | null; - userId: string | null; - width: number; -}) => { - const displayValue = userAlias || userEmail || userId; - const isDefaultAdmin = userId === DEFAULT_PROXY_ADMIN_USER_ID; - - const popoverContent = ( -
- {[ - { label: "User Alias", value: userAlias }, - { label: "User Email", value: userEmail }, - { label: "User ID", value: userId }, - ].map(({ label, value }) => ( -
- {label} - {value ? ( - - ) : ( - - - )} -
- ))} -
- ); - - const trigger = - isDefaultAdmin && !userAlias && !userEmail ? ( - - ) : ( - - ); - - return ( - - }> - {trigger} - - {popoverContent} - - ); -}; - const InfoHeader = ({ label, tooltip }: { label: string; tooltip: string }) => ( {label} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx new file mode 100644 index 00000000000..223d66d7421 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx @@ -0,0 +1,62 @@ +"use client"; + +import DefaultProxyAdminTag from "@/components/common_components/DefaultProxyAdminTag"; +import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; +import { userDetailHref } from "@/utils/entityLinks"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; + +import { IdCell } from "./id_cell"; +import { IdentityCell } from "./identity_cell"; + +export const ENTITY_CELL_TITLE_CLASSES = "font-mono text-xs font-normal"; + +interface UserPopoverCellProps { + userAlias: string | null; + userEmail: string | null; + userId: string | null; + width: number; +} + +export function UserPopoverCell({ userAlias, userEmail, userId, width }: UserPopoverCellProps) { + const displayValue = userAlias || userEmail || userId; + const isDefaultAdmin = userId === DEFAULT_PROXY_ADMIN_USER_ID; + + const popoverContent = ( +
+ {[ + { label: "User Alias", value: userAlias }, + { label: "User Email", value: userEmail }, + { label: "User ID", value: userId }, + ].map(({ label, value }) => ( +
+ {label} + {value ? ( + + ) : ( + - + )} +
+ ))} +
+ ); + + const trigger = + isDefaultAdmin && !userAlias && !userEmail ? ( + + ) : ( + + ); + + return ( + + }> + {trigger} + + {popoverContent} + + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/index.ts b/ui/litellm-dashboard/src/components/shared/table_cells/index.ts index 99b54f9aad0..34e3701eed4 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/index.ts +++ b/ui/litellm-dashboard/src/components/shared/table_cells/index.ts @@ -13,3 +13,4 @@ export { ModelsCell } from "./models_cell"; export { MoneyCell } from "./money_cell"; export { SpendBudgetCell } from "./spend_budget_cell"; export { StatusBadge, type StatusTone } from "./status_badge"; +export { UserPopoverCell, ENTITY_CELL_TITLE_CLASSES } from "./UserPopoverCell"; diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx index 0d9d0988aa1..76df0d12539 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx @@ -10,6 +10,8 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ useKeys: vi.fn(), })); +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); + vi.mock("../key_team_helpers/fetch_available_models_team_key", () => ({ getModelDisplayName: vi.fn((model: string) => model), })); @@ -384,4 +386,64 @@ describe("TeamVirtualKeysTable", () => { expect(screen.getByText("Key Info View")).toBeInTheDocument(); }); }); + + describe("entity links out of the key rows", () => { + const renderRow = async (key: KeyResponse, organization: Organization | null = null) => { + mockUseKeys.mockReturnValue({ + data: { keys: [key], total_count: 1, current_page: 1, total_pages: 1 } as KeysResponse, + isPending: false, + isFetching: false, + refetch: vi.fn(), + } as any); + renderWithProviders(); + return (await screen.findByText(key.key_alias as string)).closest("tr") as HTMLElement; + }; + + it("points the Organization ID cell at the org's detail page", async () => { + const row = await renderRow(createMockKey({ organization_id: null }), mockOrganization); + expect(within(row).getByRole("link", { name: "org-123" })).toHaveAttribute( + "href", + "/ui/organizations?org=org-123", + ); + }); + + it("points the User Email and User ID cells at the owning user's detail page", async () => { + const row = await renderRow( + createMockKey({ user_id: "user-1", user: { user_id: "user-1", user_email: "alice@example.com" } }), + ); + expect(within(row).getByRole("link", { name: "alice@example.com" })).toHaveAttribute( + "href", + "/ui/users?user=user-1", + ); + expect(within(row).getByRole("link", { name: "user-1" })).toHaveAttribute("href", "/ui/users?user=user-1"); + }); + + it("points the Created By cell at the creator's detail page", async () => { + const row = await renderRow( + createMockKey({ + created_by: "creator-1", + created_by_user: { user_id: "creator-1", user_email: "creator@example.com", user_alias: "The Creator" }, + }), + ); + expect(within(row).getByRole("link", { name: "The Creator" })).toHaveAttribute( + "href", + "/ui/users?user=creator-1", + ); + }); + + it("leaves the default_user_id placeholder unlinked in the User ID and Created By cells", async () => { + const placeholder = { user_id: "default_user_id", user_email: "admin@example.com", user_alias: "Proxy Admin" }; + const row = await renderRow( + createMockKey({ + user_id: placeholder.user_id, + user: placeholder, + created_by: placeholder.user_id, + created_by_user: placeholder, + }), + ); + expect(within(row).getByText("Default Proxy Admin")).toBeInTheDocument(); + expect(within(row).getByText("Proxy Admin")).toBeInTheDocument(); + expect(within(row).queryByRole("link", { name: "Proxy Admin" })).not.toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index b0380255b95..5b1b71e060e 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -1,8 +1,14 @@ "use client"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { SimpleTooltip } from "@/components/ui/tooltip"; -import CopyButton from "@/components/shared/CopyButton"; -import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; +import { + DateCell, + ENTITY_CELL_TITLE_CLASSES, + IdCell, + IdentityCell, + MoneyCell, + UserPopoverCell, +} from "@/components/shared/table_cells"; import { DataTable, DataTableFilterDrawer, @@ -11,8 +17,9 @@ import { DataTableToolbar, } from "@/components/shared/DataTable"; import { Badge } from "@/components/ui/badge"; -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; import { Input } from "@/components/ui/input"; +import { orgDetailHref, userDetailHref } from "@/utils/entityLinks"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { ColumnDef, ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table"; @@ -168,7 +175,15 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi header: "Organization ID", size: 140, enableSorting: false, - cell: (info) => (info.getValue() ? info.renderValue() : "-"), + cell: (info) => { + const orgId = info.getValue() as string | null; + if (!orgId) return "-"; + return ( + + + + ); + }, }, { id: "user_email", @@ -179,9 +194,14 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi cell: (info) => { const user = info.getValue() as { user_email?: string } | undefined; const value = user?.user_email; + const userId = info.row.original.user_id; return ( - {value ?? "-"} + ); }, @@ -194,10 +214,16 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi enableSorting: false, cell: (info) => { const userId = info.getValue() as string | null; - const displayValue = userId === "default_user_id" ? "Default Proxy Admin" : userId; + if (userId === DEFAULT_PROXY_ADMIN_USER_ID) { + return ; + } return ( - - {displayValue ?? "-"} + + ); }, @@ -221,53 +247,13 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi const userId = info.getValue() as string | null; if (!userId) return "-"; const { created_by_user } = info.row.original; - const userAlias = created_by_user?.user_alias ?? null; - const userEmail = created_by_user?.user_email ?? null; - const isDefaultAdmin = userId === "default_user_id"; - const displayValue = userAlias || userEmail || userId; - - const popoverContent = ( -
- {[ - { label: "User Alias", value: userAlias }, - { label: "User Email", value: userEmail }, - { label: "User ID", value: userId }, - ].map(({ label, value }) => ( -
- {label} - {value ? ( - - {value} - - - ) : ( - - - )} -
- ))} -
- ); - - if (isDefaultAdmin && !userAlias && !userEmail) { - return ( - - }> - - - {popoverContent} - - ); - } - return ( - - } - > - {displayValue} - - {popoverContent} - + ); }, }, From 06b259e0920792600c35c9951297fb83cd236c35 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 18:31:19 -0700 Subject: [PATCH 399/411] fix(ui): name the popover copy buttons after the field they copy The shared user popover copied alias, email and ID through three copy buttons that all announced themselves as "Copy ID", so a screen reader could not tell them apart. IdCell now takes the label, defaulting to the old text everywhere else. Also drops the closest("tr") the new link tests used, which put the testing-library/no-node-access budget over its ceiling, and asserts the sentinel row leaves User Email and the admin badge unlinked too. Claude-Session: https://claude.ai/code/session_01NfwfQhamRNnSqgXMUjf3h4 --- .../shared/table_cells/UserPopoverCell.tsx | 2 +- .../shared/table_cells/id_cell.test.tsx | 8 ++++++++ .../components/shared/table_cells/id_cell.tsx | 4 +++- .../team/TeamVirtualKeysTable.test.tsx | 20 ++++++++++--------- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx index 223d66d7421..eb786e45541 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/UserPopoverCell.tsx @@ -31,7 +31,7 @@ export function UserPopoverCell({ userAlias, userEmail, userId, width }: UserPop
{label} {value ? ( - + ) : ( - )} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx index 395c1815dd0..c715210fdde 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx @@ -71,6 +71,14 @@ describe("IdCell", () => { expect(rowClick).not.toHaveBeenCalled(); }); + it("names the copy button after the field it copies", async () => { + const user = userEvent.setup(); + render(); + expect(screen.queryByRole("button", { name: "Copy ID" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Copy User Email" })); + expect(copyToClipboardMock).toHaveBeenCalledWith("alice@example.com"); + }); + it("passes dataTestId through to the id element", () => { render(); expect(screen.getByTestId("key-id-cell")).toHaveTextContent("k-1"); diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx index 1b109a86106..33c7f835e64 100644 --- a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx +++ b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx @@ -15,6 +15,7 @@ interface IdCellProps { variant?: IdCellVariant; onClick?: (value: string) => void; copyable?: boolean; + copyLabel?: string; truncate?: boolean; fallback?: string; tooltip?: React.ReactNode; @@ -39,6 +40,7 @@ export function IdCell({ variant = "pill", onClick, copyable = false, + copyLabel = "Copy ID", truncate = true, fallback = "-", tooltip, @@ -80,7 +82,7 @@ export function IdCell({ {withTooltip}